2024-11-11 20:51:17,236 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-11 20:51:17,255 main DEBUG Took 0.012038 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 20:51:17,255 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 20:51:17,256 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 20:51:17,257 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 20:51:17,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,266 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 20:51:17,279 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,281 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,282 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,284 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,285 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,286 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,287 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,288 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,288 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,289 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,290 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,291 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,292 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:51:17,293 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,293 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 20:51:17,295 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:51:17,296 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 20:51:17,298 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 20:51:17,299 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 20:51:17,301 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 20:51:17,301 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 20:51:17,309 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 20:51:17,312 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 20:51:17,313 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 20:51:17,314 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 20:51:17,314 main DEBUG createAppenders(={Console}) 2024-11-11 20:51:17,315 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-11 20:51:17,315 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-11 20:51:17,316 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-11 20:51:17,316 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 20:51:17,317 main DEBUG OutputStream closed 2024-11-11 20:51:17,317 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 20:51:17,317 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 20:51:17,318 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-11 20:51:17,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 20:51:17,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 20:51:17,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 20:51:17,389 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 20:51:17,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 20:51:17,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 20:51:17,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 20:51:17,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 20:51:17,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 20:51:17,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 20:51:17,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 20:51:17,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 20:51:17,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 20:51:17,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 20:51:17,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 20:51:17,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 20:51:17,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 20:51:17,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 20:51:17,396 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 20:51:17,396 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-11 20:51:17,397 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 20:51:17,397 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-11T20:51:17,408 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-11 20:51:17,411 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 20:51:17,411 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T20:51:17,640 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993 2024-11-11T20:51:17,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T20:51:17,680 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87, deleteOnExit=true 2024-11-11T20:51:17,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T20:51:17,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/test.cache.data in system properties and HBase conf 2024-11-11T20:51:17,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T20:51:17,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir in system properties and HBase conf 2024-11-11T20:51:17,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T20:51:17,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T20:51:17,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T20:51:17,781 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T20:51:17,879 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:51:17,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T20:51:17,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T20:51:17,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T20:51:17,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T20:51:17,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T20:51:17,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T20:51:17,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T20:51:17,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T20:51:17,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T20:51:17,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/nfs.dump.dir in system properties and HBase conf 2024-11-11T20:51:17,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/java.io.tmpdir in system properties and HBase conf 2024-11-11T20:51:17,891 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T20:51:17,891 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T20:51:17,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T20:51:18,625 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T20:51:18,699 INFO [Time-limited test {}] log.Log(170): Logging initialized @2146ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T20:51:18,774 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:51:18,838 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T20:51:18,860 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T20:51:18,861 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T20:51:18,862 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T20:51:18,875 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:51:18,878 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir/,AVAILABLE} 2024-11-11T20:51:18,879 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T20:51:19,054 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/java.io.tmpdir/jetty-localhost-38881-hadoop-hdfs-3_4_1-tests_jar-_-any-15095249831660698196/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T20:51:19,059 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:38881} 2024-11-11T20:51:19,060 INFO [Time-limited test {}] server.Server(415): Started @2507ms 2024-11-11T20:51:19,358 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:51:19,366 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T20:51:19,367 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T20:51:19,367 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T20:51:19,368 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T20:51:19,369 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir/,AVAILABLE} 2024-11-11T20:51:19,370 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T20:51:19,468 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/java.io.tmpdir/jetty-localhost-35283-hadoop-hdfs-3_4_1-tests_jar-_-any-3946286529535974645/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:51:19,468 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:35283} 2024-11-11T20:51:19,469 INFO [Time-limited test {}] server.Server(415): Started @2916ms 2024-11-11T20:51:19,517 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T20:51:19,626 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:51:19,635 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T20:51:19,639 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T20:51:19,640 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T20:51:19,640 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T20:51:19,641 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir/,AVAILABLE} 2024-11-11T20:51:19,642 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T20:51:19,779 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/java.io.tmpdir/jetty-localhost-34989-hadoop-hdfs-3_4_1-tests_jar-_-any-14104233874457841240/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:51:19,780 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:34989} 2024-11-11T20:51:19,780 INFO [Time-limited test {}] server.Server(415): Started @3228ms 2024-11-11T20:51:19,782 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T20:51:19,835 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:51:19,846 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T20:51:19,847 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T20:51:19,848 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T20:51:19,848 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T20:51:19,849 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir/,AVAILABLE} 2024-11-11T20:51:19,849 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T20:51:19,918 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data2/current/BP-896987307-172.17.0.2-1731358278425/current, will proceed with Du for space computation calculation, 2024-11-11T20:51:19,918 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data4/current/BP-896987307-172.17.0.2-1731358278425/current, will proceed with Du for space computation calculation, 2024-11-11T20:51:19,918 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data1/current/BP-896987307-172.17.0.2-1731358278425/current, will proceed with Du for space computation calculation, 2024-11-11T20:51:19,918 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data3/current/BP-896987307-172.17.0.2-1731358278425/current, will proceed with Du for space computation calculation, 2024-11-11T20:51:19,962 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/java.io.tmpdir/jetty-localhost-43629-hadoop-hdfs-3_4_1-tests_jar-_-any-2502449032695931794/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:51:19,962 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T20:51:19,963 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:43629} 2024-11-11T20:51:19,963 INFO [Time-limited test {}] server.Server(415): Started @3411ms 2024-11-11T20:51:19,964 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T20:51:19,966 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T20:51:20,027 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d4d868f4f2d9143 with lease ID 0xa739bcba2bab88d5: Processing first storage report for DS-7b35a41a-1c7b-4e15-8fdf-7861a2284fc2 from datanode DatanodeRegistration(127.0.0.1:36815, datanodeUuid=dae37a18-934f-4592-8acc-44b7bbdbc324, infoPort=35225, infoSecurePort=0, ipcPort=44357, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425) 2024-11-11T20:51:20,028 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d4d868f4f2d9143 with lease ID 0xa739bcba2bab88d5: from storage DS-7b35a41a-1c7b-4e15-8fdf-7861a2284fc2 node DatanodeRegistration(127.0.0.1:36815, datanodeUuid=dae37a18-934f-4592-8acc-44b7bbdbc324, infoPort=35225, infoSecurePort=0, ipcPort=44357, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T20:51:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc76569816aca55ff with lease ID 0xa739bcba2bab88d4: Processing first storage report for DS-149c8a03-8d45-412c-8335-e18d2a7471f3 from datanode DatanodeRegistration(127.0.0.1:38355, datanodeUuid=59034bab-165e-448c-8d98-6d0f088c67f5, infoPort=41935, infoSecurePort=0, ipcPort=35877, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425) 2024-11-11T20:51:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc76569816aca55ff with lease ID 0xa739bcba2bab88d4: from storage DS-149c8a03-8d45-412c-8335-e18d2a7471f3 node DatanodeRegistration(127.0.0.1:38355, datanodeUuid=59034bab-165e-448c-8d98-6d0f088c67f5, infoPort=41935, infoSecurePort=0, ipcPort=35877, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T20:51:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d4d868f4f2d9143 with lease ID 0xa739bcba2bab88d5: Processing first storage report for DS-208bf442-bbb0-4a4a-907f-cd8ec5772743 from datanode DatanodeRegistration(127.0.0.1:36815, datanodeUuid=dae37a18-934f-4592-8acc-44b7bbdbc324, infoPort=35225, infoSecurePort=0, ipcPort=44357, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425) 2024-11-11T20:51:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d4d868f4f2d9143 with lease ID 0xa739bcba2bab88d5: from storage DS-208bf442-bbb0-4a4a-907f-cd8ec5772743 node DatanodeRegistration(127.0.0.1:36815, datanodeUuid=dae37a18-934f-4592-8acc-44b7bbdbc324, infoPort=35225, infoSecurePort=0, ipcPort=44357, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T20:51:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc76569816aca55ff with lease ID 0xa739bcba2bab88d4: Processing first storage report for DS-8b8274ce-46f2-4019-b96e-d0550ca94137 from datanode DatanodeRegistration(127.0.0.1:38355, datanodeUuid=59034bab-165e-448c-8d98-6d0f088c67f5, infoPort=41935, infoSecurePort=0, ipcPort=35877, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425) 2024-11-11T20:51:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc76569816aca55ff with lease ID 0xa739bcba2bab88d4: from storage DS-8b8274ce-46f2-4019-b96e-d0550ca94137 node DatanodeRegistration(127.0.0.1:38355, datanodeUuid=59034bab-165e-448c-8d98-6d0f088c67f5, infoPort=41935, infoSecurePort=0, ipcPort=35877, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T20:51:20,056 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data5/current/BP-896987307-172.17.0.2-1731358278425/current, will proceed with Du for space computation calculation, 2024-11-11T20:51:20,057 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data6/current/BP-896987307-172.17.0.2-1731358278425/current, will proceed with Du for space computation calculation, 2024-11-11T20:51:20,082 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T20:51:20,087 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x57e9e7e9f9da6bcd with lease ID 0xa739bcba2bab88d6: Processing first storage report for DS-b1845f9b-ff07-4be7-bc46-1cdf58d7c517 from datanode DatanodeRegistration(127.0.0.1:45587, datanodeUuid=a166485b-082b-47f0-9dd4-85e666463258, infoPort=37191, infoSecurePort=0, ipcPort=40499, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425) 2024-11-11T20:51:20,087 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x57e9e7e9f9da6bcd with lease ID 0xa739bcba2bab88d6: from storage DS-b1845f9b-ff07-4be7-bc46-1cdf58d7c517 node DatanodeRegistration(127.0.0.1:45587, datanodeUuid=a166485b-082b-47f0-9dd4-85e666463258, infoPort=37191, infoSecurePort=0, ipcPort=40499, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T20:51:20,087 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x57e9e7e9f9da6bcd with lease ID 0xa739bcba2bab88d6: Processing first storage report for DS-806c58ac-8194-4bdf-9249-529ffd765da2 from datanode DatanodeRegistration(127.0.0.1:45587, datanodeUuid=a166485b-082b-47f0-9dd4-85e666463258, infoPort=37191, infoSecurePort=0, ipcPort=40499, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425) 2024-11-11T20:51:20,088 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x57e9e7e9f9da6bcd with lease ID 0xa739bcba2bab88d6: from storage DS-806c58ac-8194-4bdf-9249-529ffd765da2 node DatanodeRegistration(127.0.0.1:45587, datanodeUuid=a166485b-082b-47f0-9dd4-85e666463258, infoPort=37191, infoSecurePort=0, ipcPort=40499, storageInfo=lv=-57;cid=testClusterID;nsid=1398794686;c=1731358278425), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T20:51:20,288 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993 2024-11-11T20:51:20,363 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/zookeeper_0, clientPort=57196, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T20:51:20,372 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57196 2024-11-11T20:51:20,382 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:20,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:20,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741825_1001 (size=7) 2024-11-11T20:51:20,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741825_1001 (size=7) 2024-11-11T20:51:20,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741825_1001 (size=7) 2024-11-11T20:51:21,018 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a with version=8 2024-11-11T20:51:21,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/hbase-staging 2024-11-11T20:51:21,095 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T20:51:21,293 INFO [Time-limited test {}] client.ConnectionUtils(128): master/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:51:21,302 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,302 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,307 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:51:21,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:51:21,425 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T20:51:21,477 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T20:51:21,485 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T20:51:21,488 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:51:21,511 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 35037 (auto-detected) 2024-11-11T20:51:21,512 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T20:51:21,529 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38185 2024-11-11T20:51:21,548 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38185 connecting to ZooKeeper ensemble=127.0.0.1:57196 2024-11-11T20:51:21,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:381850x0, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:51:21,576 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38185-0x100308e27080000 connected 2024-11-11T20:51:21,603 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:21,607 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:21,617 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:51:21,621 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a, hbase.cluster.distributed=false 2024-11-11T20:51:21,645 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:51:21,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38185 2024-11-11T20:51:21,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38185 2024-11-11T20:51:21,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38185 2024-11-11T20:51:21,653 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38185 2024-11-11T20:51:21,654 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38185 2024-11-11T20:51:21,755 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:51:21,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,756 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:51:21,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:51:21,759 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:51:21,762 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:51:21,762 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36213 2024-11-11T20:51:21,764 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36213 connecting to ZooKeeper ensemble=127.0.0.1:57196 2024-11-11T20:51:21,765 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:21,767 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:21,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362130x0, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:51:21,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:51:21,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36213-0x100308e27080001 connected 2024-11-11T20:51:21,776 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:51:21,783 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:51:21,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:51:21,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:51:21,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36213 2024-11-11T20:51:21,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36213 2024-11-11T20:51:21,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36213 2024-11-11T20:51:21,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36213 2024-11-11T20:51:21,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36213 2024-11-11T20:51:21,808 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:51:21,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,809 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:51:21,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:51:21,809 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:51:21,809 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:51:21,810 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38917 2024-11-11T20:51:21,812 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38917 connecting to ZooKeeper ensemble=127.0.0.1:57196 2024-11-11T20:51:21,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:21,815 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:21,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389170x0, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:51:21,820 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:51:21,820 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38917-0x100308e27080002 connected 2024-11-11T20:51:21,820 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:51:21,821 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:51:21,822 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:51:21,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:51:21,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38917 2024-11-11T20:51:21,825 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38917 2024-11-11T20:51:21,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38917 2024-11-11T20:51:21,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38917 2024-11-11T20:51:21,826 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38917 2024-11-11T20:51:21,843 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:51:21,843 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,843 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,843 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:51:21,844 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:51:21,844 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:51:21,844 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:51:21,844 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:51:21,845 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36001 2024-11-11T20:51:21,847 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36001 connecting to ZooKeeper ensemble=127.0.0.1:57196 2024-11-11T20:51:21,848 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:21,850 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:21,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360010x0, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:51:21,856 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360010x0, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:51:21,856 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36001-0x100308e27080003 connected 2024-11-11T20:51:21,857 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:51:21,861 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:51:21,862 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:51:21,864 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:51:21,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36001 2024-11-11T20:51:21,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36001 2024-11-11T20:51:21,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36001 2024-11-11T20:51:21,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36001 2024-11-11T20:51:21,870 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36001 2024-11-11T20:51:21,883 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;51ca66f7ee3c:38185 2024-11-11T20:51:21,884 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:21,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:51:21,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:51:21,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:51:21,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:51:21,891 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:51:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:51:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:51:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:21,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:21,913 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T20:51:21,914 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/51ca66f7ee3c,38185,1731358281139 from backup master directory 2024-11-11T20:51:21,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:21,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:51:21,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:51:21,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:51:21,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:51:21,917 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:51:21,918 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:21,920 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T20:51:21,921 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T20:51:21,976 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/hbase.id] with ID: 35edf637-a5f4-4537-9f2c-797160efd3f6 2024-11-11T20:51:21,976 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/.tmp/hbase.id 2024-11-11T20:51:21,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741826_1002 (size=42) 2024-11-11T20:51:21,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741826_1002 (size=42) 2024-11-11T20:51:21,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741826_1002 (size=42) 2024-11-11T20:51:21,992 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/.tmp/hbase.id]:[hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/hbase.id] 2024-11-11T20:51:22,035 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:51:22,040 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T20:51:22,058 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-11T20:51:22,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741827_1003 (size=196) 2024-11-11T20:51:22,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741827_1003 (size=196) 2024-11-11T20:51:22,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741827_1003 (size=196) 2024-11-11T20:51:22,094 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:51:22,096 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T20:51:22,108 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:51:22,112 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:51:22,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741828_1004 (size=1189) 2024-11-11T20:51:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741828_1004 (size=1189) 2024-11-11T20:51:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741828_1004 (size=1189) 2024-11-11T20:51:22,165 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store 2024-11-11T20:51:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741829_1005 (size=34) 2024-11-11T20:51:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741829_1005 (size=34) 2024-11-11T20:51:22,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741829_1005 (size=34) 2024-11-11T20:51:22,191 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T20:51:22,194 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:22,196 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:51:22,196 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:51:22,196 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:51:22,199 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:51:22,199 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:51:22,199 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:51:22,201 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731358282195Disabling compacts and flushes for region at 1731358282195Disabling writes for close at 1731358282199 (+4 ms)Writing region close event to WAL at 1731358282199Closed at 1731358282199 2024-11-11T20:51:22,203 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/.initializing 2024-11-11T20:51:22,204 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/WALs/51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:22,213 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:51:22,229 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C38185%2C1731358281139, suffix=, logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/WALs/51ca66f7ee3c,38185,1731358281139, archiveDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/oldWALs, maxLogs=10 2024-11-11T20:51:22,252 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/WALs/51ca66f7ee3c,38185,1731358281139/51ca66f7ee3c%2C38185%2C1731358281139.1731358282234, exclude list is [], retry=0 2024-11-11T20:51:22,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36815,DS-7b35a41a-1c7b-4e15-8fdf-7861a2284fc2,DISK] 2024-11-11T20:51:22,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b1845f9b-ff07-4be7-bc46-1cdf58d7c517,DISK] 2024-11-11T20:51:22,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38355,DS-149c8a03-8d45-412c-8335-e18d2a7471f3,DISK] 2024-11-11T20:51:22,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T20:51:22,310 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/WALs/51ca66f7ee3c,38185,1731358281139/51ca66f7ee3c%2C38185%2C1731358281139.1731358282234 2024-11-11T20:51:22,311 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37191:37191),(127.0.0.1/127.0.0.1:35225:35225),(127.0.0.1/127.0.0.1:41935:41935)] 2024-11-11T20:51:22,311 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:51:22,311 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:22,314 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,315 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T20:51:22,376 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:22,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,382 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T20:51:22,383 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:22,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,387 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T20:51:22,387 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:22,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T20:51:22,391 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:22,392 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,396 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,397 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,402 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,403 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,406 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:51:22,410 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:51:22,415 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:22,416 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66315158, jitterRate=-0.011827141046524048}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:51:22,424 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731358282327Initializing all the Stores at 1731358282329 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358282329Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358282330 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358282330Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358282330Cleaning up temporary data from old regions at 1731358282403 (+73 ms)Region opened successfully at 1731358282424 (+21 ms) 2024-11-11T20:51:22,426 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T20:51:22,457 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e380f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:51:22,484 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T20:51:22,494 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:22,494 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:22,497 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:22,498 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T20:51:22,503 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T20:51:22,503 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T20:51:22,528 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T20:51:22,536 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T20:51:22,538 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T20:51:22,540 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T20:51:22,541 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T20:51:22,542 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T20:51:22,545 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T20:51:22,548 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T20:51:22,549 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T20:51:22,551 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T20:51:22,551 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T20:51:22,567 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T20:51:22,568 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T20:51:22,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:51:22,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:51:22,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:51:22,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:51:22,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,575 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=51ca66f7ee3c,38185,1731358281139, sessionid=0x100308e27080000, setting cluster-up flag (Was=false) 2024-11-11T20:51:22,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,590 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T20:51:22,591 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:22,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:22,603 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T20:51:22,605 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:22,612 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T20:51:22,673 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(746): ClusterId : 35edf637-a5f4-4537-9f2c-797160efd3f6 2024-11-11T20:51:22,674 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(746): ClusterId : 35edf637-a5f4-4537-9f2c-797160efd3f6 2024-11-11T20:51:22,675 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(746): ClusterId : 35edf637-a5f4-4537-9f2c-797160efd3f6 2024-11-11T20:51:22,677 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:51:22,677 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:51:22,677 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:51:22,682 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:51:22,682 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:51:22,682 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:51:22,682 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:51:22,682 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:51:22,682 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:51:22,686 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:51:22,686 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:51:22,686 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:51:22,687 DEBUG [RS:0;51ca66f7ee3c:36213 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27171fa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:51:22,687 DEBUG [RS:2;51ca66f7ee3c:36001 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4581f2a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:51:22,687 DEBUG [RS:1;51ca66f7ee3c:38917 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fb2a2ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:51:22,688 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:22,702 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T20:51:22,703 DEBUG [RS:0;51ca66f7ee3c:36213 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;51ca66f7ee3c:36213 2024-11-11T20:51:22,704 DEBUG [RS:1;51ca66f7ee3c:38917 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;51ca66f7ee3c:38917 2024-11-11T20:51:22,706 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:51:22,706 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:51:22,707 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:51:22,707 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:51:22,707 DEBUG [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:51:22,707 DEBUG [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:51:22,710 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,38185,1731358281139 with port=36213, startcode=1731358281724 2024-11-11T20:51:22,710 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,38185,1731358281139 with port=38917, startcode=1731358281807 2024-11-11T20:51:22,711 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T20:51:22,712 DEBUG [RS:2;51ca66f7ee3c:36001 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;51ca66f7ee3c:36001 2024-11-11T20:51:22,713 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:51:22,713 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:51:22,713 DEBUG [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:51:22,714 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,38185,1731358281139 with port=36001, startcode=1731358281842 2024-11-11T20:51:22,719 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 51ca66f7ee3c,38185,1731358281139 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T20:51:22,725 DEBUG [RS:0;51ca66f7ee3c:36213 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:51:22,725 DEBUG [RS:1;51ca66f7ee3c:38917 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:51:22,725 DEBUG [RS:2;51ca66f7ee3c:36001 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:51:22,729 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:51:22,730 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:51:22,730 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:51:22,730 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:51:22,730 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/51ca66f7ee3c:0, corePoolSize=10, maxPoolSize=10 2024-11-11T20:51:22,731 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,731 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:51:22,731 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,732 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358312732 2024-11-11T20:51:22,734 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T20:51:22,736 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T20:51:22,737 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:22,738 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T20:51:22,740 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T20:51:22,741 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T20:51:22,741 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T20:51:22,742 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T20:51:22,742 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,745 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,745 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T20:51:22,750 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T20:51:22,751 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T20:51:22,752 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T20:51:22,753 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T20:51:22,754 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T20:51:22,756 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358282755,5,FailOnTimeoutGroup] 2024-11-11T20:51:22,757 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358282756,5,FailOnTimeoutGroup] 2024-11-11T20:51:22,758 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,758 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T20:51:22,760 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,760 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,779 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58909, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:51:22,779 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43661, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:51:22,779 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52235, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:51:22,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741831_1007 (size=1321) 2024-11-11T20:51:22,828 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:22,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741831_1007 (size=1321) 2024-11-11T20:51:22,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741831_1007 (size=1321) 2024-11-11T20:51:22,831 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:22,831 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T20:51:22,832 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:22,845 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:22,845 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:22,849 DEBUG [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:22,849 DEBUG [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45383 2024-11-11T20:51:22,850 DEBUG [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:51:22,851 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:22,851 DEBUG [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:22,851 DEBUG [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45383 2024-11-11T20:51:22,852 DEBUG [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:51:22,852 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:22,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741832_1008 (size=32) 2024-11-11T20:51:22,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741832_1008 (size=32) 2024-11-11T20:51:22,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:51:22,857 DEBUG [RS:1;51ca66f7ee3c:38917 {}] zookeeper.ZKUtil(111): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:22,857 WARN [RS:1;51ca66f7ee3c:38917 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:51:22,857 INFO [RS:1;51ca66f7ee3c:38917 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:51:22,858 DEBUG [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:22,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741832_1008 (size=32) 2024-11-11T20:51:22,859 DEBUG [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:22,859 DEBUG [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45383 2024-11-11T20:51:22,859 DEBUG [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:51:22,859 DEBUG [RS:2;51ca66f7ee3c:36001 {}] zookeeper.ZKUtil(111): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:22,859 WARN [RS:2;51ca66f7ee3c:36001 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:51:22,859 INFO [RS:2;51ca66f7ee3c:36001 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:51:22,860 DEBUG [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:22,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:22,862 DEBUG [RS:0;51ca66f7ee3c:36213 {}] zookeeper.ZKUtil(111): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:22,863 WARN [RS:0;51ca66f7ee3c:36213 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:51:22,863 INFO [RS:0;51ca66f7ee3c:36213 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:51:22,863 DEBUG [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:22,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T20:51:22,864 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,36001,1731358281842] 2024-11-11T20:51:22,865 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,38917,1731358281807] 2024-11-11T20:51:22,865 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,36213,1731358281724] 2024-11-11T20:51:22,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:51:22,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:22,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T20:51:22,880 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:51:22,880 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:22,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T20:51:22,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:51:22,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:22,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T20:51:22,891 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:51:22,891 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:22,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:22,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:51:22,894 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:51:22,894 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:51:22,894 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:51:22,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740 2024-11-11T20:51:22,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740 2024-11-11T20:51:22,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:51:22,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:51:22,900 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:51:22,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:51:22,914 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:22,916 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59225245, jitterRate=-0.11747507750988007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:51:22,918 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731358282860Initializing all the Stores at 1731358282863 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358282863Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358282863Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358282863Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358282864 (+1 ms)Cleaning up temporary data from old regions at 1731358282899 (+35 ms)Region opened successfully at 1731358282917 (+18 ms) 2024-11-11T20:51:22,918 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:51:22,918 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:51:22,918 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:51:22,918 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:51:22,918 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:51:22,919 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:51:22,920 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:51:22,921 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:51:22,931 INFO [RS:0;51ca66f7ee3c:36213 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:51:22,931 INFO [RS:1;51ca66f7ee3c:38917 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:51:22,932 INFO [RS:2;51ca66f7ee3c:36001 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:51:22,932 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,932 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,932 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,933 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:51:22,933 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:51:22,933 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358282918Disabling compacts and flushes for region at 1731358282918Disabling writes for close at 1731358282918Writing region close event to WAL at 1731358282933 (+15 ms)Closed at 1731358282933 2024-11-11T20:51:22,937 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:22,937 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T20:51:22,939 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:51:22,940 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:51:22,941 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:51:22,942 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,942 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,943 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,943 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,943 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,943 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,943 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,944 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,944 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,944 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,944 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,944 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,944 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:51:22,944 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:51:22,944 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,944 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,944 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,945 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:51:22,945 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:51:22,945 DEBUG [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:51:22,945 DEBUG [RS:2;51ca66f7ee3c:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:51:22,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T20:51:22,949 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:51:22,951 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,951 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,951 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,951 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,951 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:51:22,951 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,951 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,36213,1731358281724-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:51:22,951 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,952 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,952 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,952 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,952 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,952 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,952 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:51:22,952 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,953 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,953 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,953 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,953 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,953 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:51:22,953 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:51:22,953 DEBUG [RS:1;51ca66f7ee3c:38917 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:51:22,957 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,957 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,957 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,957 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,958 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,958 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,36001,1731358281842-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:51:22,965 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,965 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,965 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,965 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,965 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,966 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38917,1731358281807-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:51:22,971 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:22,976 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T20:51:22,978 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:51:22,980 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,36213,1731358281724-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,981 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,981 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.Replication(171): 51ca66f7ee3c,36213,1731358281724 started 2024-11-11T20:51:22,981 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:51:22,981 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,36001,1731358281842-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,982 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,982 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.Replication(171): 51ca66f7ee3c,36001,1731358281842 started 2024-11-11T20:51:22,993 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:51:22,993 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38917,1731358281807-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,993 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:22,993 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.Replication(171): 51ca66f7ee3c,38917,1731358281807 started 2024-11-11T20:51:23,010 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,010 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,011 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,36213,1731358281724, RpcServer on 51ca66f7ee3c/172.17.0.2:36213, sessionid=0x100308e27080001 2024-11-11T20:51:23,011 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,36001,1731358281842, RpcServer on 51ca66f7ee3c/172.17.0.2:36001, sessionid=0x100308e27080003 2024-11-11T20:51:23,012 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,012 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:51:23,012 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:51:23,012 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,38917,1731358281807, RpcServer on 51ca66f7ee3c/172.17.0.2:38917, sessionid=0x100308e27080002 2024-11-11T20:51:23,012 DEBUG [RS:2;51ca66f7ee3c:36001 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:23,012 DEBUG [RS:0;51ca66f7ee3c:36213 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:23,012 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:51:23,012 DEBUG [RS:1;51ca66f7ee3c:38917 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:23,012 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,36001,1731358281842' 2024-11-11T20:51:23,012 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,36213,1731358281724' 2024-11-11T20:51:23,013 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,38917,1731358281807' 2024-11-11T20:51:23,013 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:51:23,013 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:51:23,013 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:51:23,014 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:51:23,014 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:51:23,014 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:51:23,014 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:51:23,014 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:51:23,014 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:51:23,014 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:51:23,014 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:51:23,014 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:51:23,015 DEBUG [RS:0;51ca66f7ee3c:36213 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:23,015 DEBUG [RS:1;51ca66f7ee3c:38917 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:23,015 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,36213,1731358281724' 2024-11-11T20:51:23,015 DEBUG [RS:2;51ca66f7ee3c:36001 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:23,015 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:51:23,015 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,38917,1731358281807' 2024-11-11T20:51:23,015 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,36001,1731358281842' 2024-11-11T20:51:23,015 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:51:23,015 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:51:23,015 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:51:23,015 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:51:23,016 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:51:23,016 DEBUG [RS:0;51ca66f7ee3c:36213 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:51:23,016 INFO [RS:0;51ca66f7ee3c:36213 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:51:23,016 INFO [RS:0;51ca66f7ee3c:36213 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:51:23,016 DEBUG [RS:1;51ca66f7ee3c:38917 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:51:23,016 INFO [RS:1;51ca66f7ee3c:38917 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:51:23,016 INFO [RS:1;51ca66f7ee3c:38917 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:51:23,016 DEBUG [RS:2;51ca66f7ee3c:36001 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:51:23,016 INFO [RS:2;51ca66f7ee3c:36001 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:51:23,017 INFO [RS:2;51ca66f7ee3c:36001 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:51:23,122 INFO [RS:1;51ca66f7ee3c:38917 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:51:23,122 INFO [RS:0;51ca66f7ee3c:36213 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:51:23,122 INFO [RS:2;51ca66f7ee3c:36001 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:51:23,126 INFO [RS:0;51ca66f7ee3c:36213 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C36213%2C1731358281724, suffix=, logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36213,1731358281724, archiveDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs, maxLogs=32 2024-11-11T20:51:23,126 INFO [RS:2;51ca66f7ee3c:36001 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C36001%2C1731358281842, suffix=, logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36001,1731358281842, archiveDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs, maxLogs=32 2024-11-11T20:51:23,126 INFO [RS:1;51ca66f7ee3c:38917 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C38917%2C1731358281807, suffix=, logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807, archiveDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs, maxLogs=32 2024-11-11T20:51:23,127 WARN [51ca66f7ee3c:38185 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T20:51:23,146 DEBUG [RS:2;51ca66f7ee3c:36001 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36001,1731358281842/51ca66f7ee3c%2C36001%2C1731358281842.1731358283129, exclude list is [], retry=0 2024-11-11T20:51:23,148 DEBUG [RS:0;51ca66f7ee3c:36213 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36213,1731358281724/51ca66f7ee3c%2C36213%2C1731358281724.1731358283129, exclude list is [], retry=0 2024-11-11T20:51:23,148 DEBUG [RS:1;51ca66f7ee3c:38917 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, exclude list is [], retry=0 2024-11-11T20:51:23,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38355,DS-149c8a03-8d45-412c-8335-e18d2a7471f3,DISK] 2024-11-11T20:51:23,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36815,DS-7b35a41a-1c7b-4e15-8fdf-7861a2284fc2,DISK] 2024-11-11T20:51:23,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38355,DS-149c8a03-8d45-412c-8335-e18d2a7471f3,DISK] 2024-11-11T20:51:23,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36815,DS-7b35a41a-1c7b-4e15-8fdf-7861a2284fc2,DISK] 2024-11-11T20:51:23,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38355,DS-149c8a03-8d45-412c-8335-e18d2a7471f3,DISK] 2024-11-11T20:51:23,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b1845f9b-ff07-4be7-bc46-1cdf58d7c517,DISK] 2024-11-11T20:51:23,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b1845f9b-ff07-4be7-bc46-1cdf58d7c517,DISK] 2024-11-11T20:51:23,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36815,DS-7b35a41a-1c7b-4e15-8fdf-7861a2284fc2,DISK] 2024-11-11T20:51:23,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b1845f9b-ff07-4be7-bc46-1cdf58d7c517,DISK] 2024-11-11T20:51:23,163 INFO [RS:2;51ca66f7ee3c:36001 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36001,1731358281842/51ca66f7ee3c%2C36001%2C1731358281842.1731358283129 2024-11-11T20:51:23,163 INFO [RS:1;51ca66f7ee3c:38917 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129 2024-11-11T20:51:23,165 DEBUG [RS:2;51ca66f7ee3c:36001 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41935:41935),(127.0.0.1/127.0.0.1:35225:35225),(127.0.0.1/127.0.0.1:37191:37191)] 2024-11-11T20:51:23,166 DEBUG [RS:1;51ca66f7ee3c:38917 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41935:41935),(127.0.0.1/127.0.0.1:37191:37191),(127.0.0.1/127.0.0.1:35225:35225)] 2024-11-11T20:51:23,169 INFO [RS:0;51ca66f7ee3c:36213 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36213,1731358281724/51ca66f7ee3c%2C36213%2C1731358281724.1731358283129 2024-11-11T20:51:23,172 DEBUG [RS:0;51ca66f7ee3c:36213 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41935:41935),(127.0.0.1/127.0.0.1:35225:35225),(127.0.0.1/127.0.0.1:37191:37191)] 2024-11-11T20:51:23,384 DEBUG [51ca66f7ee3c:38185 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-11T20:51:23,395 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:51:23,401 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:51:23,401 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:51:23,401 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-11T20:51:23,401 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:51:23,402 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:51:23,402 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-11T20:51:23,402 INFO [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:51:23,402 INFO [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:51:23,402 INFO [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-11T20:51:23,402 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:51:23,409 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:23,415 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,38917,1731358281807, state=OPENING 2024-11-11T20:51:23,419 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T20:51:23,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:23,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:23,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:23,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:23,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:23,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:23,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:23,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:23,423 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:23,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:23,610 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:51:23,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56411, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:51:23,625 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T20:51:23,625 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:51:23,625 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T20:51:23,629 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C38917%2C1731358281807.meta, suffix=.meta, logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807, archiveDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs, maxLogs=32 2024-11-11T20:51:23,645 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, exclude list is [], retry=0 2024-11-11T20:51:23,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36815,DS-7b35a41a-1c7b-4e15-8fdf-7861a2284fc2,DISK] 2024-11-11T20:51:23,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38355,DS-149c8a03-8d45-412c-8335-e18d2a7471f3,DISK] 2024-11-11T20:51:23,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b1845f9b-ff07-4be7-bc46-1cdf58d7c517,DISK] 2024-11-11T20:51:23,652 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta 2024-11-11T20:51:23,652 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41935:41935),(127.0.0.1/127.0.0.1:35225:35225),(127.0.0.1/127.0.0.1:37191:37191)] 2024-11-11T20:51:23,652 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:51:23,654 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T20:51:23,657 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T20:51:23,661 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T20:51:23,665 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T20:51:23,665 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:23,666 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T20:51:23,666 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T20:51:23,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T20:51:23,670 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:51:23,671 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:23,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:23,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T20:51:23,673 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:51:23,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:23,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:23,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T20:51:23,675 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:51:23,675 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:23,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:23,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T20:51:23,677 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:51:23,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:23,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:23,679 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:51:23,680 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740 2024-11-11T20:51:23,682 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740 2024-11-11T20:51:23,685 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:51:23,685 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:51:23,686 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:51:23,688 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:51:23,690 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59618211, jitterRate=-0.11161942780017853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:51:23,690 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T20:51:23,692 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731358283666Writing region info on filesystem at 1731358283666Initializing all the Stores at 1731358283668 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358283668Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358283669 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358283669Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358283669Cleaning up temporary data from old regions at 1731358283685 (+16 ms)Running coprocessor post-open hooks at 1731358283690 (+5 ms)Region opened successfully at 1731358283691 (+1 ms) 2024-11-11T20:51:23,698 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731358283599 2024-11-11T20:51:23,709 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T20:51:23,710 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T20:51:23,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:23,714 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,38917,1731358281807, state=OPEN 2024-11-11T20:51:23,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:23,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:23,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:23,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:23,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:23,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:23,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:23,717 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:23,718 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:23,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T20:51:23,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,38917,1731358281807 in 293 msec 2024-11-11T20:51:23,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T20:51:23,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 782 msec 2024-11-11T20:51:23,735 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:23,735 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T20:51:23,754 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:51:23,755 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1] 2024-11-11T20:51:23,776 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:51:23,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:51:23,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1560 sec 2024-11-11T20:51:23,798 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731358283798, completionTime=-1 2024-11-11T20:51:23,802 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-11T20:51:23,802 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:23,834 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-11T20:51:23,834 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358343834 2024-11-11T20:51:23,834 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358403834 2024-11-11T20:51:23,835 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-11T20:51:23,836 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:23,845 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38185,1731358281139-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,846 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38185,1731358281139-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,846 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38185,1731358281139-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,848 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-51ca66f7ee3c:38185, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,849 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,850 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:23,857 DEBUG [master/51ca66f7ee3c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T20:51:23,878 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.959sec 2024-11-11T20:51:23,879 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T20:51:23,880 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T20:51:23,882 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T20:51:23,882 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T20:51:23,883 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T20:51:23,884 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38185,1731358281139-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:51:23,884 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38185,1731358281139-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T20:51:23,889 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T20:51:23,890 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T20:51:23,891 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38185,1731358281139-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:51:24,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7818ae86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:51:24,013 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T20:51:24,013 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T20:51:24,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38185,-1 for getting cluster id 2024-11-11T20:51:24,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:51:24,034 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '35edf637-a5f4-4537-9f2c-797160efd3f6' 2024-11-11T20:51:24,037 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:51:24,037 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "35edf637-a5f4-4537-9f2c-797160efd3f6" 2024-11-11T20:51:24,040 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f7e62c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:51:24,040 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38185,-1] 2024-11-11T20:51:24,043 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:51:24,045 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:24,047 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:51:24,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ff0df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:51:24,050 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:51:24,056 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1] 2024-11-11T20:51:24,057 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:51:24,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:51:24,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:24,081 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:51:24,084 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:24,087 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@667b3226 2024-11-11T20:51:24,087 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:51:24,090 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:51:24,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:51:24,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:24,107 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:51:24,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-11T20:51:24,109 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:24,111 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:51:24,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:51:24,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741837_1013 (size=358) 2024-11-11T20:51:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741837_1013 (size=358) 2024-11-11T20:51:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741837_1013 (size=358) 2024-11-11T20:51:24,129 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f749f9c70b714192bc4197ca43f0d35e, NAME => 'test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,129 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 57a71397de63e8cb71cb808d7888d245, NAME => 'test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,130 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c574684bc1a84767063943424c84e3f2, NAME => 'test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741838_1014 (size=45) 2024-11-11T20:51:24,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741838_1014 (size=45) 2024-11-11T20:51:24,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741838_1014 (size=45) 2024-11-11T20:51:24,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741839_1015 (size=45) 2024-11-11T20:51:24,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741839_1015 (size=45) 2024-11-11T20:51:24,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741839_1015 (size=45) 2024-11-11T20:51:24,148 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,149 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 57a71397de63e8cb71cb808d7888d245, disabling compactions & flushes 2024-11-11T20:51:24,149 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:24,149 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:24,149 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. after waiting 0 ms 2024-11-11T20:51:24,149 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:24,149 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:24,149 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,149 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 57a71397de63e8cb71cb808d7888d245: Waiting for close lock at 1731358284149Disabling compacts and flushes for region at 1731358284149Disabling writes for close at 1731358284149Writing region close event to WAL at 1731358284149Closed at 1731358284149 2024-11-11T20:51:24,149 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c574684bc1a84767063943424c84e3f2, disabling compactions & flushes 2024-11-11T20:51:24,149 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:24,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741840_1016 (size=42) 2024-11-11T20:51:24,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741840_1016 (size=42) 2024-11-11T20:51:24,150 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:24,150 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. after waiting 0 ms 2024-11-11T20:51:24,150 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:24,150 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => eef5c8934a3876cee983f4197f34ce96, NAME => 'test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,150 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:24,150 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c574684bc1a84767063943424c84e3f2: Waiting for close lock at 1731358284149Disabling compacts and flushes for region at 1731358284149Disabling writes for close at 1731358284150 (+1 ms)Writing region close event to WAL at 1731358284150Closed at 1731358284150 2024-11-11T20:51:24,151 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => caaf3dccceaaa47724f8b76041286a47, NAME => 'test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741840_1016 (size=42) 2024-11-11T20:51:24,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f749f9c70b714192bc4197ca43f0d35e, disabling compactions & flushes 2024-11-11T20:51:24,152 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e. 2024-11-11T20:51:24,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e. 2024-11-11T20:51:24,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e. after waiting 0 ms 2024-11-11T20:51:24,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e. 2024-11-11T20:51:24,152 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e. 2024-11-11T20:51:24,153 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f749f9c70b714192bc4197ca43f0d35e: Waiting for close lock at 1731358284152Disabling compacts and flushes for region at 1731358284152Disabling writes for close at 1731358284152Writing region close event to WAL at 1731358284152Closed at 1731358284152 2024-11-11T20:51:24,153 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 48dede73c5017e00f25348e75653b620, NAME => 'test,fff,1731358284091.48dede73c5017e00f25348e75653b620.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741841_1017 (size=45) 2024-11-11T20:51:24,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741841_1017 (size=45) 2024-11-11T20:51:24,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741841_1017 (size=45) 2024-11-11T20:51:24,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741842_1018 (size=45) 2024-11-11T20:51:24,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741842_1018 (size=45) 2024-11-11T20:51:24,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741842_1018 (size=45) 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing caaf3dccceaaa47724f8b76041286a47, disabling compactions & flushes 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing eef5c8934a3876cee983f4197f34ce96, disabling compactions & flushes 2024-11-11T20:51:24,165 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:24,165 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. after waiting 0 ms 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. after waiting 0 ms 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:24,165 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:24,165 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for eef5c8934a3876cee983f4197f34ce96: Waiting for close lock at 1731358284165Disabling compacts and flushes for region at 1731358284165Disabling writes for close at 1731358284165Writing region close event to WAL at 1731358284165Closed at 1731358284165 2024-11-11T20:51:24,165 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for caaf3dccceaaa47724f8b76041286a47: Waiting for close lock at 1731358284165Disabling compacts and flushes for region at 1731358284165Disabling writes for close at 1731358284165Writing region close event to WAL at 1731358284165Closed at 1731358284165 2024-11-11T20:51:24,166 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fec0270188c1b9e90ffcce7addd0cf30, NAME => 'test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,166 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9dfd755f05156eb757bfa3d4fdc2b62a, NAME => 'test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741843_1019 (size=45) 2024-11-11T20:51:24,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741843_1019 (size=45) 2024-11-11T20:51:24,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741843_1019 (size=45) 2024-11-11T20:51:24,172 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1731358284091.48dede73c5017e00f25348e75653b620.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,172 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 48dede73c5017e00f25348e75653b620, disabling compactions & flushes 2024-11-11T20:51:24,172 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:24,172 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:24,172 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731358284091.48dede73c5017e00f25348e75653b620. after waiting 0 ms 2024-11-11T20:51:24,172 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:24,173 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:24,173 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 48dede73c5017e00f25348e75653b620: Waiting for close lock at 1731358284172Disabling compacts and flushes for region at 1731358284172Disabling writes for close at 1731358284172Writing region close event to WAL at 1731358284172Closed at 1731358284172 2024-11-11T20:51:24,173 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 885e30f0459dab7e5a3e27492bed3f2a, NAME => 'test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741844_1020 (size=45) 2024-11-11T20:51:24,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741844_1020 (size=45) 2024-11-11T20:51:24,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741844_1020 (size=45) 2024-11-11T20:51:24,187 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,187 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9dfd755f05156eb757bfa3d4fdc2b62a, disabling compactions & flushes 2024-11-11T20:51:24,187 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:24,187 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:24,187 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. after waiting 0 ms 2024-11-11T20:51:24,187 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:24,188 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:24,188 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9dfd755f05156eb757bfa3d4fdc2b62a: Waiting for close lock at 1731358284187Disabling compacts and flushes for region at 1731358284187Disabling writes for close at 1731358284187Writing region close event to WAL at 1731358284188 (+1 ms)Closed at 1731358284188 2024-11-11T20:51:24,188 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4a56b479883cb03a610f18bcad1bfcc3, NAME => 'test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741845_1021 (size=45) 2024-11-11T20:51:24,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741845_1021 (size=45) 2024-11-11T20:51:24,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741845_1021 (size=45) 2024-11-11T20:51:24,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fec0270188c1b9e90ffcce7addd0cf30, disabling compactions & flushes 2024-11-11T20:51:24,194 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:24,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:24,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. after waiting 0 ms 2024-11-11T20:51:24,194 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:24,195 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:24,195 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fec0270188c1b9e90ffcce7addd0cf30: Waiting for close lock at 1731358284194Disabling compacts and flushes for region at 1731358284194Disabling writes for close at 1731358284194Writing region close event to WAL at 1731358284195 (+1 ms)Closed at 1731358284195 2024-11-11T20:51:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741846_1022 (size=45) 2024-11-11T20:51:24,196 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 96c53e851b8fac62f296d140948ca3ee, NAME => 'test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741846_1022 (size=45) 2024-11-11T20:51:24,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741846_1022 (size=45) 2024-11-11T20:51:24,198 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 885e30f0459dab7e5a3e27492bed3f2a, disabling compactions & flushes 2024-11-11T20:51:24,199 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. after waiting 0 ms 2024-11-11T20:51:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:24,199 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:24,199 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 885e30f0459dab7e5a3e27492bed3f2a: Waiting for close lock at 1731358284198Disabling compacts and flushes for region at 1731358284198Disabling writes for close at 1731358284199 (+1 ms)Writing region close event to WAL at 1731358284199Closed at 1731358284199 2024-11-11T20:51:24,200 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 084ef38b701ab7eeaa42a673a92ec1d6, NAME => 'test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741847_1023 (size=45) 2024-11-11T20:51:24,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741847_1023 (size=45) 2024-11-11T20:51:24,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741847_1023 (size=45) 2024-11-11T20:51:24,214 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,214 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4a56b479883cb03a610f18bcad1bfcc3, disabling compactions & flushes 2024-11-11T20:51:24,214 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:24,214 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:24,214 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. after waiting 0 ms 2024-11-11T20:51:24,214 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:24,215 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:24,215 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4a56b479883cb03a610f18bcad1bfcc3: Waiting for close lock at 1731358284214Disabling compacts and flushes for region at 1731358284214Disabling writes for close at 1731358284214Writing region close event to WAL at 1731358284215 (+1 ms)Closed at 1731358284215 2024-11-11T20:51:24,215 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0b414e49a28b2cae7a6a20bc170cff4f, NAME => 'test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741848_1024 (size=45) 2024-11-11T20:51:24,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741848_1024 (size=45) 2024-11-11T20:51:24,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741848_1024 (size=45) 2024-11-11T20:51:24,218 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,218 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 96c53e851b8fac62f296d140948ca3ee, disabling compactions & flushes 2024-11-11T20:51:24,218 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:24,218 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:24,218 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. after waiting 0 ms 2024-11-11T20:51:24,218 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:24,218 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:24,219 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 96c53e851b8fac62f296d140948ca3ee: Waiting for close lock at 1731358284218Disabling compacts and flushes for region at 1731358284218Disabling writes for close at 1731358284218Writing region close event to WAL at 1731358284218Closed at 1731358284218 2024-11-11T20:51:24,220 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e6f7fe1f29c3816146fbdd9ef0338b39, NAME => 'test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:51:24,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741849_1025 (size=45) 2024-11-11T20:51:24,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741849_1025 (size=45) 2024-11-11T20:51:24,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741849_1025 (size=45) 2024-11-11T20:51:24,230 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,230 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 084ef38b701ab7eeaa42a673a92ec1d6, disabling compactions & flushes 2024-11-11T20:51:24,230 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:24,230 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:24,231 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. after waiting 0 ms 2024-11-11T20:51:24,231 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:24,231 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:24,231 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 084ef38b701ab7eeaa42a673a92ec1d6: Waiting for close lock at 1731358284230Disabling compacts and flushes for region at 1731358284230Disabling writes for close at 1731358284231 (+1 ms)Writing region close event to WAL at 1731358284231Closed at 1731358284231 2024-11-11T20:51:24,231 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0a298d35593aa3585cf7f54bd6d37af4, NAME => 'test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741850_1026 (size=45) 2024-11-11T20:51:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741850_1026 (size=45) 2024-11-11T20:51:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741851_1027 (size=45) 2024-11-11T20:51:24,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741851_1027 (size=45) 2024-11-11T20:51:24,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741851_1027 (size=45) 2024-11-11T20:51:24,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741850_1026 (size=45) 2024-11-11T20:51:24,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741852_1028 (size=45) 2024-11-11T20:51:24,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741852_1028 (size=45) 2024-11-11T20:51:24,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741852_1028 (size=45) 2024-11-11T20:51:24,252 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 0a298d35593aa3585cf7f54bd6d37af4, disabling compactions & flushes 2024-11-11T20:51:24,253 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:24,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:24,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. after waiting 0 ms 2024-11-11T20:51:24,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:24,253 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:24,253 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0a298d35593aa3585cf7f54bd6d37af4: Waiting for close lock at 1731358284253Disabling compacts and flushes for region at 1731358284253Disabling writes for close at 1731358284253Writing region close event to WAL at 1731358284253Closed at 1731358284253 2024-11-11T20:51:24,254 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bce7c26d0c06e82319102e4a44901ffa, NAME => 'test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741853_1029 (size=45) 2024-11-11T20:51:24,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741853_1029 (size=45) 2024-11-11T20:51:24,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741853_1029 (size=45) 2024-11-11T20:51:24,268 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,268 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bce7c26d0c06e82319102e4a44901ffa, disabling compactions & flushes 2024-11-11T20:51:24,268 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:24,268 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:24,268 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. after waiting 0 ms 2024-11-11T20:51:24,268 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:24,268 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:24,268 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bce7c26d0c06e82319102e4a44901ffa: Waiting for close lock at 1731358284268Disabling compacts and flushes for region at 1731358284268Disabling writes for close at 1731358284268Writing region close event to WAL at 1731358284268Closed at 1731358284268 2024-11-11T20:51:24,269 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1c7ea4c5037945c7e8566334a0de9a50, NAME => 'test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741854_1030 (size=45) 2024-11-11T20:51:24,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741854_1030 (size=45) 2024-11-11T20:51:24,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741854_1030 (size=45) 2024-11-11T20:51:24,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1c7ea4c5037945c7e8566334a0de9a50, disabling compactions & flushes 2024-11-11T20:51:24,286 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. 2024-11-11T20:51:24,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. 2024-11-11T20:51:24,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. after waiting 0 ms 2024-11-11T20:51:24,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. 2024-11-11T20:51:24,286 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. 2024-11-11T20:51:24,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1c7ea4c5037945c7e8566334a0de9a50: Waiting for close lock at 1731358284285Disabling compacts and flushes for region at 1731358284285Disabling writes for close at 1731358284286 (+1 ms)Writing region close event to WAL at 1731358284286Closed at 1731358284286 2024-11-11T20:51:24,286 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0e0eaf6d4b3f10b135b25df3514191b, NAME => 'test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741855_1031 (size=45) 2024-11-11T20:51:24,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741855_1031 (size=45) 2024-11-11T20:51:24,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741855_1031 (size=45) 2024-11-11T20:51:24,299 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,299 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d0e0eaf6d4b3f10b135b25df3514191b, disabling compactions & flushes 2024-11-11T20:51:24,299 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:24,299 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:24,300 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. after waiting 0 ms 2024-11-11T20:51:24,300 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:24,300 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:24,300 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0e0eaf6d4b3f10b135b25df3514191b: Waiting for close lock at 1731358284299Disabling compacts and flushes for region at 1731358284299Disabling writes for close at 1731358284300 (+1 ms)Writing region close event to WAL at 1731358284300Closed at 1731358284300 2024-11-11T20:51:24,300 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 43d3520c27ced34912ce528d771ce465, NAME => 'test,sss,1731358284091.43d3520c27ced34912ce528d771ce465.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741856_1032 (size=45) 2024-11-11T20:51:24,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741856_1032 (size=45) 2024-11-11T20:51:24,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741856_1032 (size=45) 2024-11-11T20:51:24,316 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1731358284091.43d3520c27ced34912ce528d771ce465.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,316 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 43d3520c27ced34912ce528d771ce465, disabling compactions & flushes 2024-11-11T20:51:24,316 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:24,316 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:24,316 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. after waiting 0 ms 2024-11-11T20:51:24,316 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:24,316 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:24,316 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 43d3520c27ced34912ce528d771ce465: Waiting for close lock at 1731358284316Disabling compacts and flushes for region at 1731358284316Disabling writes for close at 1731358284316Writing region close event to WAL at 1731358284316Closed at 1731358284316 2024-11-11T20:51:24,317 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07a697e4da0c9ded22ab8014e5ed7708, NAME => 'test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741857_1033 (size=45) 2024-11-11T20:51:24,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741857_1033 (size=45) 2024-11-11T20:51:24,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741857_1033 (size=45) 2024-11-11T20:51:24,328 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,328 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 07a697e4da0c9ded22ab8014e5ed7708, disabling compactions & flushes 2024-11-11T20:51:24,328 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:24,328 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:24,328 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. after waiting 0 ms 2024-11-11T20:51:24,328 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:24,329 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:24,329 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07a697e4da0c9ded22ab8014e5ed7708: Waiting for close lock at 1731358284328Disabling compacts and flushes for region at 1731358284328Disabling writes for close at 1731358284328Writing region close event to WAL at 1731358284328Closed at 1731358284328 2024-11-11T20:51:24,329 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5acc05cce6a62ba139f97712c529e120, NAME => 'test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741858_1034 (size=45) 2024-11-11T20:51:24,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741858_1034 (size=45) 2024-11-11T20:51:24,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741858_1034 (size=45) 2024-11-11T20:51:24,340 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,340 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5acc05cce6a62ba139f97712c529e120, disabling compactions & flushes 2024-11-11T20:51:24,340 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:24,340 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:24,340 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. after waiting 0 ms 2024-11-11T20:51:24,340 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:24,341 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:24,341 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5acc05cce6a62ba139f97712c529e120: Waiting for close lock at 1731358284340Disabling compacts and flushes for region at 1731358284340Disabling writes for close at 1731358284340Writing region close event to WAL at 1731358284340Closed at 1731358284340 2024-11-11T20:51:24,341 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f43990b9e210c622161ccc0992d6c911, NAME => 'test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741859_1035 (size=45) 2024-11-11T20:51:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741859_1035 (size=45) 2024-11-11T20:51:24,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741859_1035 (size=45) 2024-11-11T20:51:24,353 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,353 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f43990b9e210c622161ccc0992d6c911, disabling compactions & flushes 2024-11-11T20:51:24,353 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:24,353 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:24,354 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. after waiting 0 ms 2024-11-11T20:51:24,354 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:24,354 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:24,354 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f43990b9e210c622161ccc0992d6c911: Waiting for close lock at 1731358284353Disabling compacts and flushes for region at 1731358284353Disabling writes for close at 1731358284354 (+1 ms)Writing region close event to WAL at 1731358284354Closed at 1731358284354 2024-11-11T20:51:24,354 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 201d6d0f92a6319d987b626f2b7bad8b, NAME => 'test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741860_1036 (size=45) 2024-11-11T20:51:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741860_1036 (size=45) 2024-11-11T20:51:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741860_1036 (size=45) 2024-11-11T20:51:24,365 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,365 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 201d6d0f92a6319d987b626f2b7bad8b, disabling compactions & flushes 2024-11-11T20:51:24,366 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b. 2024-11-11T20:51:24,366 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b. 2024-11-11T20:51:24,366 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b. after waiting 0 ms 2024-11-11T20:51:24,366 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b. 2024-11-11T20:51:24,366 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b. 2024-11-11T20:51:24,366 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 201d6d0f92a6319d987b626f2b7bad8b: Waiting for close lock at 1731358284365Disabling compacts and flushes for region at 1731358284365Disabling writes for close at 1731358284366 (+1 ms)Writing region close event to WAL at 1731358284366Closed at 1731358284366 2024-11-11T20:51:24,366 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d8b6777ce113fcb1df22e25ffb49623a, NAME => 'test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741861_1037 (size=45) 2024-11-11T20:51:24,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741861_1037 (size=45) 2024-11-11T20:51:24,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741861_1037 (size=45) 2024-11-11T20:51:24,378 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,378 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d8b6777ce113fcb1df22e25ffb49623a, disabling compactions & flushes 2024-11-11T20:51:24,378 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:24,378 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:24,378 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. after waiting 0 ms 2024-11-11T20:51:24,378 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:24,378 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:24,378 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d8b6777ce113fcb1df22e25ffb49623a: Waiting for close lock at 1731358284378Disabling compacts and flushes for region at 1731358284378Disabling writes for close at 1731358284378Writing region close event to WAL at 1731358284378Closed at 1731358284378 2024-11-11T20:51:24,379 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd0c630a3962dac6e1875877d78d1d43, NAME => 'test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741862_1038 (size=45) 2024-11-11T20:51:24,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741862_1038 (size=45) 2024-11-11T20:51:24,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741862_1038 (size=45) 2024-11-11T20:51:24,391 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,391 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bd0c630a3962dac6e1875877d78d1d43, disabling compactions & flushes 2024-11-11T20:51:24,391 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:24,391 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:24,391 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. after waiting 0 ms 2024-11-11T20:51:24,391 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:24,391 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:24,391 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd0c630a3962dac6e1875877d78d1d43: Waiting for close lock at 1731358284391Disabling compacts and flushes for region at 1731358284391Disabling writes for close at 1731358284391Writing region close event to WAL at 1731358284391Closed at 1731358284391 2024-11-11T20:51:24,392 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d414ef171f23d90cb3b6da2b30e93f36, NAME => 'test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a 2024-11-11T20:51:24,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741863_1039 (size=42) 2024-11-11T20:51:24,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741863_1039 (size=42) 2024-11-11T20:51:24,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741863_1039 (size=42) 2024-11-11T20:51:24,403 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,403 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d414ef171f23d90cb3b6da2b30e93f36, disabling compactions & flushes 2024-11-11T20:51:24,403 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:24,403 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:24,403 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. after waiting 0 ms 2024-11-11T20:51:24,403 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:24,403 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:24,403 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d414ef171f23d90cb3b6da2b30e93f36: Waiting for close lock at 1731358284403Disabling compacts and flushes for region at 1731358284403Disabling writes for close at 1731358284403Writing region close event to WAL at 1731358284403Closed at 1731358284403 2024-11-11T20:51:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:51:24,645 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,645 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:24,645 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e6f7fe1f29c3816146fbdd9ef0338b39, disabling compactions & flushes 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0b414e49a28b2cae7a6a20bc170cff4f, disabling compactions & flushes 2024-11-11T20:51:24,646 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:24,646 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. after waiting 0 ms 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. after waiting 0 ms 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:24,646 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:24,646 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0b414e49a28b2cae7a6a20bc170cff4f: Waiting for close lock at 1731358284645Disabling compacts and flushes for region at 1731358284645Disabling writes for close at 1731358284646 (+1 ms)Writing region close event to WAL at 1731358284646Closed at 1731358284646 2024-11-11T20:51:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e6f7fe1f29c3816146fbdd9ef0338b39: Waiting for close lock at 1731358284645Disabling compacts and flushes for region at 1731358284645Disabling writes for close at 1731358284646 (+1 ms)Writing region close event to WAL at 1731358284646Closed at 1731358284646 2024-11-11T20:51:24,649 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:51:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731358284091.48dede73c5017e00f25348e75653b620.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731358284091.43d3520c27ced34912ce528d771ce465.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,660 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,661 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,661 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731358284650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358284650"}]},"ts":"1731358284650"} 2024-11-11T20:51:24,700 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-11T20:51:24,702 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:51:24,704 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358284702"}]},"ts":"1731358284702"} 2024-11-11T20:51:24,708 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-11T20:51:24,709 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:51:24,725 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:51:24,725 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:51:24,725 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-11T20:51:24,725 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:51:24,725 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:51:24,725 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-11T20:51:24,725 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:51:24,725 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:51:24,725 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-11T20:51:24,725 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:51:24,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c574684bc1a84767063943424c84e3f2, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57a71397de63e8cb71cb808d7888d245, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eef5c8934a3876cee983f4197f34ce96, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=caaf3dccceaaa47724f8b76041286a47, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fec0270188c1b9e90ffcce7addd0cf30, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9dfd755f05156eb757bfa3d4fdc2b62a, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a56b479883cb03a610f18bcad1bfcc3, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=96c53e851b8fac62f296d140948ca3ee, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0b414e49a28b2cae7a6a20bc170cff4f, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e6f7fe1f29c3816146fbdd9ef0338b39, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0a298d35593aa3585cf7f54bd6d37af4, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bce7c26d0c06e82319102e4a44901ffa, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d0e0eaf6d4b3f10b135b25df3514191b, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=43d3520c27ced34912ce528d771ce465, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5acc05cce6a62ba139f97712c529e120, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f43990b9e210c622161ccc0992d6c911, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d8b6777ce113fcb1df22e25ffb49623a, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bd0c630a3962dac6e1875877d78d1d43, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN}] 2024-11-11T20:51:24,738 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:24,739 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,38917,1731358281807; forceNewPlan=false, retain=false 2024-11-11T20:51:24,741 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bd0c630a3962dac6e1875877d78d1d43, ASSIGN 2024-11-11T20:51:24,742 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bd0c630a3962dac6e1875877d78d1d43, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:51:24,744 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d8b6777ce113fcb1df22e25ffb49623a, ASSIGN 2024-11-11T20:51:24,745 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d8b6777ce113fcb1df22e25ffb49623a, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,747 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:24,748 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,38917,1731358281807; forceNewPlan=false, retain=false 2024-11-11T20:51:24,750 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f43990b9e210c622161ccc0992d6c911, ASSIGN 2024-11-11T20:51:24,752 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f43990b9e210c622161ccc0992d6c911, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,753 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5acc05cce6a62ba139f97712c529e120, ASSIGN 2024-11-11T20:51:24,755 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5acc05cce6a62ba139f97712c529e120, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,756 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:24,758 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,38917,1731358281807; forceNewPlan=false, retain=false 2024-11-11T20:51:24,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=43d3520c27ced34912ce528d771ce465, ASSIGN 2024-11-11T20:51:24,761 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=43d3520c27ced34912ce528d771ce465, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,763 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d0e0eaf6d4b3f10b135b25df3514191b, ASSIGN 2024-11-11T20:51:24,765 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d0e0eaf6d4b3f10b135b25df3514191b, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,766 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:24,768 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,38917,1731358281807; forceNewPlan=false, retain=false 2024-11-11T20:51:24,770 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bce7c26d0c06e82319102e4a44901ffa, ASSIGN 2024-11-11T20:51:24,772 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bce7c26d0c06e82319102e4a44901ffa, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,773 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0a298d35593aa3585cf7f54bd6d37af4, ASSIGN 2024-11-11T20:51:24,775 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0a298d35593aa3585cf7f54bd6d37af4, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e6f7fe1f29c3816146fbdd9ef0338b39, ASSIGN 2024-11-11T20:51:24,778 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e6f7fe1f29c3816146fbdd9ef0338b39, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,779 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0b414e49a28b2cae7a6a20bc170cff4f, ASSIGN 2024-11-11T20:51:24,780 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0b414e49a28b2cae7a6a20bc170cff4f, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,782 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:24,783 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,38917,1731358281807; forceNewPlan=false, retain=false 2024-11-11T20:51:24,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=96c53e851b8fac62f296d140948ca3ee, ASSIGN 2024-11-11T20:51:24,786 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=96c53e851b8fac62f296d140948ca3ee, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,787 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4a56b479883cb03a610f18bcad1bfcc3, ASSIGN 2024-11-11T20:51:24,789 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4a56b479883cb03a610f18bcad1bfcc3, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,791 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:24,792 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,38917,1731358281807; forceNewPlan=false, retain=false 2024-11-11T20:51:24,793 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9dfd755f05156eb757bfa3d4fdc2b62a, ASSIGN 2024-11-11T20:51:24,795 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9dfd755f05156eb757bfa3d4fdc2b62a, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,797 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fec0270188c1b9e90ffcce7addd0cf30, ASSIGN 2024-11-11T20:51:24,799 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fec0270188c1b9e90ffcce7addd0cf30, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,800 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:24,802 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,38917,1731358281807; forceNewPlan=false, retain=false 2024-11-11T20:51:24,803 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=caaf3dccceaaa47724f8b76041286a47, ASSIGN 2024-11-11T20:51:24,805 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=caaf3dccceaaa47724f8b76041286a47, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,806 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eef5c8934a3876cee983f4197f34ce96, ASSIGN 2024-11-11T20:51:24,807 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eef5c8934a3876cee983f4197f34ce96, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,809 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57a71397de63e8cb71cb808d7888d245, ASSIGN 2024-11-11T20:51:24,810 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=57a71397de63e8cb71cb808d7888d245, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36001,1731358281842; forceNewPlan=false, retain=false 2024-11-11T20:51:24,811 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c574684bc1a84767063943424c84e3f2, ASSIGN 2024-11-11T20:51:24,813 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c574684bc1a84767063943424c84e3f2, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,36213,1731358281724; forceNewPlan=false, retain=false 2024-11-11T20:51:24,815 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:24,816 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,38917,1731358281807; forceNewPlan=false, retain=false 2024-11-11T20:51:24,895 INFO [51ca66f7ee3c:38185 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-11T20:51:24,897 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:24,898 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:24,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:24,900 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:24,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:24,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN because future has completed 2024-11-11T20:51:24,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:24,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:24,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN because future has completed 2024-11-11T20:51:24,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN because future has completed 2024-11-11T20:51:24,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN because future has completed 2024-11-11T20:51:24,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN because future has completed 2024-11-11T20:51:24,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7ea4c5037945c7e8566334a0de9a50, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:24,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN because future has completed 2024-11-11T20:51:24,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:24,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:24,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:24,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:24,939 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=48dede73c5017e00f25348e75653b620, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:24,940 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:24,940 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=bd0c630a3962dac6e1875877d78d1d43, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,941 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f43990b9e210c622161ccc0992d6c911, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,942 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=43d3520c27ced34912ce528d771ce465, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,943 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=bce7c26d0c06e82319102e4a44901ffa, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN because future has completed 2024-11-11T20:51:24,944 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:24,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN because future has completed 2024-11-11T20:51:24,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bd0c630a3962dac6e1875877d78d1d43, ASSIGN because future has completed 2024-11-11T20:51:24,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f43990b9e210c622161ccc0992d6c911, ASSIGN because future has completed 2024-11-11T20:51:24,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=43d3520c27ced34912ce528d771ce465, ASSIGN because future has completed 2024-11-11T20:51:24,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43d3520c27ced34912ce528d771ce465, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:24,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bce7c26d0c06e82319102e4a44901ffa, ASSIGN because future has completed 2024-11-11T20:51:24,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e6f7fe1f29c3816146fbdd9ef0338b39, ASSIGN because future has completed 2024-11-11T20:51:24,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6f7fe1f29c3816146fbdd9ef0338b39, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:24,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure bce7c26d0c06e82319102e4a44901ffa, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:24,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43990b9e210c622161ccc0992d6c911, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:24,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd0c630a3962dac6e1875877d78d1d43, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:24,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,38917,1731358281807}] 2024-11-11T20:51:24,986 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=96c53e851b8fac62f296d140948ca3ee, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,987 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=caaf3dccceaaa47724f8b76041286a47, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c574684bc1a84767063943424c84e3f2, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:24,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:24,989 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5acc05cce6a62ba139f97712c529e120, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:24,989 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:24,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=96c53e851b8fac62f296d140948ca3ee, ASSIGN because future has completed 2024-11-11T20:51:24,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96c53e851b8fac62f296d140948ca3ee, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:24,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fec0270188c1b9e90ffcce7addd0cf30, ASSIGN because future has completed 2024-11-11T20:51:24,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=caaf3dccceaaa47724f8b76041286a47, ASSIGN because future has completed 2024-11-11T20:51:24,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c574684bc1a84767063943424c84e3f2, ASSIGN because future has completed 2024-11-11T20:51:24,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d8b6777ce113fcb1df22e25ffb49623a, ASSIGN because future has completed 2024-11-11T20:51:24,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c574684bc1a84767063943424c84e3f2, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:24,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5acc05cce6a62ba139f97712c529e120, ASSIGN because future has completed 2024-11-11T20:51:24,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d0e0eaf6d4b3f10b135b25df3514191b, ASSIGN because future has completed 2024-11-11T20:51:25,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0e0eaf6d4b3f10b135b25df3514191b, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acc05cce6a62ba139f97712c529e120, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8b6777ce113fcb1df22e25ffb49623a, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure caaf3dccceaaa47724f8b76041286a47, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:25,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec0270188c1b9e90ffcce7addd0cf30, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:25,026 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,029 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=eef5c8934a3876cee983f4197f34ce96, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=57a71397de63e8cb71cb808d7888d245, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0a298d35593aa3585cf7f54bd6d37af4, ASSIGN because future has completed 2024-11-11T20:51:25,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a298d35593aa3585cf7f54bd6d37af4, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0b414e49a28b2cae7a6a20bc170cff4f, ASSIGN because future has completed 2024-11-11T20:51:25,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4a56b479883cb03a610f18bcad1bfcc3, ASSIGN because future has completed 2024-11-11T20:51:25,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9dfd755f05156eb757bfa3d4fdc2b62a, ASSIGN because future has completed 2024-11-11T20:51:25,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eef5c8934a3876cee983f4197f34ce96, ASSIGN because future has completed 2024-11-11T20:51:25,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure eef5c8934a3876cee983f4197f34ce96, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=57a71397de63e8cb71cb808d7888d245, ASSIGN because future has completed 2024-11-11T20:51:25,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a71397de63e8cb71cb808d7888d245, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dfd755f05156eb757bfa3d4fdc2b62a, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a56b479883cb03a610f18bcad1bfcc3, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b414e49a28b2cae7a6a20bc170cff4f, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:25,064 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:25,065 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => d414ef171f23d90cb3b6da2b30e93f36, NAME => 'test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-11T20:51:25,065 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,065 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,065 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,065 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,068 INFO [StoreOpener-d414ef171f23d90cb3b6da2b30e93f36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,070 INFO [StoreOpener-d414ef171f23d90cb3b6da2b30e93f36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d414ef171f23d90cb3b6da2b30e93f36 columnFamilyName family 2024-11-11T20:51:25,070 DEBUG [StoreOpener-d414ef171f23d90cb3b6da2b30e93f36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,072 INFO [StoreOpener-d414ef171f23d90cb3b6da2b30e93f36-1 {}] regionserver.HStore(327): Store=d414ef171f23d90cb3b6da2b30e93f36/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,072 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,073 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,074 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,074 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,074 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,077 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,079 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d414ef171f23d90cb3b6da2b30e93f36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,080 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened d414ef171f23d90cb3b6da2b30e93f36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63207760, jitterRate=-0.05813097953796387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,080 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:25,081 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for d414ef171f23d90cb3b6da2b30e93f36: Running coprocessor pre-open hook at 1731358285066Writing region info on filesystem at 1731358285066Initializing all the Stores at 1731358285067 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285067Cleaning up temporary data from old regions at 1731358285074 (+7 ms)Running coprocessor post-open hooks at 1731358285080 (+6 ms)Region opened successfully at 1731358285081 (+1 ms) 2024-11-11T20:51:25,083 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36., pid=31, masterSystemTime=1731358285058 2024-11-11T20:51:25,086 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:25,086 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:25,087 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:25,087 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 48dede73c5017e00f25348e75653b620, NAME => 'test,fff,1731358284091.48dede73c5017e00f25348e75653b620.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-11T20:51:25,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:25,088 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,088 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,fff,1731358284091.48dede73c5017e00f25348e75653b620.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,088 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,088 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,38917,1731358281807 because future has completed 2024-11-11T20:51:25,090 INFO [StoreOpener-48dede73c5017e00f25348e75653b620-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,093 INFO [StoreOpener-48dede73c5017e00f25348e75653b620-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48dede73c5017e00f25348e75653b620 columnFamilyName family 2024-11-11T20:51:25,093 DEBUG [StoreOpener-48dede73c5017e00f25348e75653b620-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,094 INFO [StoreOpener-48dede73c5017e00f25348e75653b620-1 {}] regionserver.HStore(327): Store=48dede73c5017e00f25348e75653b620/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,094 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-11T20:51:25,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,38917,1731358281807 in 189 msec 2024-11-11T20:51:25,095 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,096 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,097 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,097 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN in 368 msec 2024-11-11T20:51:25,099 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,102 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/48dede73c5017e00f25348e75653b620/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,103 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 48dede73c5017e00f25348e75653b620; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59436253, jitterRate=-0.11433081328868866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,103 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:25,103 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 48dede73c5017e00f25348e75653b620: Running coprocessor pre-open hook at 1731358285088Writing region info on filesystem at 1731358285088Initializing all the Stores at 1731358285090 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285090Cleaning up temporary data from old regions at 1731358285097 (+7 ms)Running coprocessor post-open hooks at 1731358285103 (+6 ms)Region opened successfully at 1731358285103 2024-11-11T20:51:25,104 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731358284091.48dede73c5017e00f25348e75653b620., pid=37, masterSystemTime=1731358285058 2024-11-11T20:51:25,107 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:25,107 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:25,107 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. 2024-11-11T20:51:25,107 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c7ea4c5037945c7e8566334a0de9a50, NAME => 'test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-11T20:51:25,107 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=48dede73c5017e00f25348e75653b620, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:25,107 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,108 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,108 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,108 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,110 INFO [StoreOpener-1c7ea4c5037945c7e8566334a0de9a50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,110 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:51:25,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,38917,1731358281807 because future has completed 2024-11-11T20:51:25,113 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47849, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:51:25,113 INFO [StoreOpener-1c7ea4c5037945c7e8566334a0de9a50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c7ea4c5037945c7e8566334a0de9a50 columnFamilyName family 2024-11-11T20:51:25,113 DEBUG [StoreOpener-1c7ea4c5037945c7e8566334a0de9a50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,114 INFO [StoreOpener-1c7ea4c5037945c7e8566334a0de9a50-1 {}] regionserver.HStore(327): Store=1c7ea4c5037945c7e8566334a0de9a50/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,114 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,116 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,117 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=10 2024-11-11T20:51:25,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,38917,1731358281807 in 169 msec 2024-11-11T20:51:25,118 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,118 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,120 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:25,120 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 96c53e851b8fac62f296d140948ca3ee, NAME => 'test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-11T20:51:25,121 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,121 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,121 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,122 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,122 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN in 390 msec 2024-11-11T20:51:25,128 INFO [StoreOpener-96c53e851b8fac62f296d140948ca3ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,129 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/1c7ea4c5037945c7e8566334a0de9a50/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,130 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 1c7ea4c5037945c7e8566334a0de9a50; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75107542, jitterRate=0.11918959021568298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,131 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:25,131 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 1c7ea4c5037945c7e8566334a0de9a50: Running coprocessor pre-open hook at 1731358285108Writing region info on filesystem at 1731358285108Initializing all the Stores at 1731358285109 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285109Cleaning up temporary data from old regions at 1731358285118 (+9 ms)Running coprocessor post-open hooks at 1731358285131 (+13 ms)Region opened successfully at 1731358285131 2024-11-11T20:51:25,132 INFO [StoreOpener-96c53e851b8fac62f296d140948ca3ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96c53e851b8fac62f296d140948ca3ee columnFamilyName family 2024-11-11T20:51:25,132 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50., pid=32, masterSystemTime=1731358285058 2024-11-11T20:51:25,132 DEBUG [StoreOpener-96c53e851b8fac62f296d140948ca3ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,133 INFO [StoreOpener-96c53e851b8fac62f296d140948ca3ee-1 {}] regionserver.HStore(327): Store=96c53e851b8fac62f296d140948ca3ee/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,133 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,135 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,136 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. 2024-11-11T20:51:25,136 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,136 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. 2024-11-11T20:51:25,136 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b. 2024-11-11T20:51:25,136 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 201d6d0f92a6319d987b626f2b7bad8b, NAME => 'test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-11T20:51:25,137 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,137 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,137 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,137 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,137 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,137 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,137 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:25,140 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,140 INFO [StoreOpener-201d6d0f92a6319d987b626f2b7bad8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7ea4c5037945c7e8566334a0de9a50, server=51ca66f7ee3c,38917,1731358281807 because future has completed 2024-11-11T20:51:25,143 INFO [StoreOpener-201d6d0f92a6319d987b626f2b7bad8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 201d6d0f92a6319d987b626f2b7bad8b columnFamilyName family 2024-11-11T20:51:25,143 DEBUG [StoreOpener-201d6d0f92a6319d987b626f2b7bad8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,144 INFO [StoreOpener-201d6d0f92a6319d987b626f2b7bad8b-1 {}] regionserver.HStore(327): Store=201d6d0f92a6319d987b626f2b7bad8b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,145 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,146 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,147 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/96c53e851b8fac62f296d140948ca3ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,147 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,148 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 96c53e851b8fac62f296d140948ca3ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68268417, jitterRate=0.01727868616580963}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,148 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:25,148 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,148 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,148 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 96c53e851b8fac62f296d140948ca3ee: Running coprocessor pre-open hook at 1731358285122Writing region info on filesystem at 1731358285122Initializing all the Stores at 1731358285123 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285123Cleaning up temporary data from old regions at 1731358285137 (+14 ms)Running coprocessor post-open hooks at 1731358285148 (+11 ms)Region opened successfully at 1731358285148 2024-11-11T20:51:25,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-11-11T20:51:25,149 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee., pid=44, masterSystemTime=1731358285110 2024-11-11T20:51:25,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c7ea4c5037945c7e8566334a0de9a50, server=51ca66f7ee3c,38917,1731358281807 in 234 msec 2024-11-11T20:51:25,151 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,153 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:25,153 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:25,153 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:25,154 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => bce7c26d0c06e82319102e4a44901ffa, NAME => 'test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-11T20:51:25,154 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,154 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,155 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,155 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:51:25,155 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN in 422 msec 2024-11-11T20:51:25,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58937, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:51:25,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,162 INFO [StoreOpener-bce7c26d0c06e82319102e4a44901ffa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,163 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:25,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96c53e851b8fac62f296d140948ca3ee, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,164 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/201d6d0f92a6319d987b626f2b7bad8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,165 INFO [StoreOpener-bce7c26d0c06e82319102e4a44901ffa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bce7c26d0c06e82319102e4a44901ffa columnFamilyName family 2024-11-11T20:51:25,165 DEBUG [StoreOpener-bce7c26d0c06e82319102e4a44901ffa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,165 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 5acc05cce6a62ba139f97712c529e120, NAME => 'test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-11T20:51:25,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,166 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 201d6d0f92a6319d987b626f2b7bad8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71600773, jitterRate=0.06693466007709503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 201d6d0f92a6319d987b626f2b7bad8b 2024-11-11T20:51:25,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,166 INFO [StoreOpener-bce7c26d0c06e82319102e4a44901ffa-1 {}] regionserver.HStore(327): Store=bce7c26d0c06e82319102e4a44901ffa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 201d6d0f92a6319d987b626f2b7bad8b: Running coprocessor pre-open hook at 1731358285137Writing region info on filesystem at 1731358285137Initializing all the Stores at 1731358285139 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285139Cleaning up temporary data from old regions at 1731358285148 (+9 ms)Running coprocessor post-open hooks at 1731358285166 (+18 ms)Region opened successfully at 1731358285166 2024-11-11T20:51:25,166 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,167 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,168 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,169 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b., pid=36, masterSystemTime=1731358285058 2024-11-11T20:51:25,169 INFO [StoreOpener-5acc05cce6a62ba139f97712c529e120-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,169 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,169 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,173 INFO [StoreOpener-5acc05cce6a62ba139f97712c529e120-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5acc05cce6a62ba139f97712c529e120 columnFamilyName family 2024-11-11T20:51:25,173 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,174 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b. 2024-11-11T20:51:25,174 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,www,1731358284091.201d6d0f92a6319d987b626f2b7bad8b. 2024-11-11T20:51:25,174 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:25,174 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 07a697e4da0c9ded22ab8014e5ed7708, NAME => 'test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-11T20:51:25,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=15 2024-11-11T20:51:25,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 96c53e851b8fac62f296d140948ca3ee, server=51ca66f7ee3c,36213,1731358281724 in 178 msec 2024-11-11T20:51:25,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,175 DEBUG [StoreOpener-5acc05cce6a62ba139f97712c529e120-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,177 INFO [StoreOpener-5acc05cce6a62ba139f97712c529e120-1 {}] regionserver.HStore(327): Store=5acc05cce6a62ba139f97712c529e120/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,177 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,178 INFO [StoreOpener-07a697e4da0c9ded22ab8014e5ed7708-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=96c53e851b8fac62f296d140948ca3ee, ASSIGN in 447 msec 2024-11-11T20:51:25,179 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,180 INFO [StoreOpener-07a697e4da0c9ded22ab8014e5ed7708-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07a697e4da0c9ded22ab8014e5ed7708 columnFamilyName family 2024-11-11T20:51:25,180 DEBUG [StoreOpener-07a697e4da0c9ded22ab8014e5ed7708-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,180 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,181 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:25,181 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/bce7c26d0c06e82319102e4a44901ffa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,181 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,181 INFO [StoreOpener-07a697e4da0c9ded22ab8014e5ed7708-1 {}] regionserver.HStore(327): Store=07a697e4da0c9ded22ab8014e5ed7708/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,181 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,182 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,183 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened bce7c26d0c06e82319102e4a44901ffa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71442231, jitterRate=0.06457220017910004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,183 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,183 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:25,184 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for bce7c26d0c06e82319102e4a44901ffa: Running coprocessor pre-open hook at 1731358285155Writing region info on filesystem at 1731358285155Initializing all the Stores at 1731358285156 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285156Cleaning up temporary data from old regions at 1731358285170 (+14 ms)Running coprocessor post-open hooks at 1731358285183 (+13 ms)Region opened successfully at 1731358285183 2024-11-11T20:51:25,184 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,185 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,38917,1731358281807 because future has completed 2024-11-11T20:51:25,185 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,186 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa., pid=40, masterSystemTime=1731358285110 2024-11-11T20:51:25,187 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,189 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,190 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:25,190 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:25,190 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:25,191 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => bd0c630a3962dac6e1875877d78d1d43, NAME => 'test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-11T20:51:25,192 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,192 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,192 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,192 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,193 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/5acc05cce6a62ba139f97712c529e120/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,194 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/07a697e4da0c9ded22ab8014e5ed7708/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,195 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 5acc05cce6a62ba139f97712c529e120; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66474744, jitterRate=-0.009449124336242676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,195 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 07a697e4da0c9ded22ab8014e5ed7708; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75096856, jitterRate=0.11903035640716553}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=27 2024-11-11T20:51:25,195 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:25,195 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:25,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,38917,1731358281807 in 254 msec 2024-11-11T20:51:25,195 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 5acc05cce6a62ba139f97712c529e120: Running coprocessor pre-open hook at 1731358285167Writing region info on filesystem at 1731358285167Initializing all the Stores at 1731358285168 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285168Cleaning up temporary data from old regions at 1731358285182 (+14 ms)Running coprocessor post-open hooks at 1731358285195 (+13 ms)Region opened successfully at 1731358285195 2024-11-11T20:51:25,195 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 07a697e4da0c9ded22ab8014e5ed7708: Running coprocessor pre-open hook at 1731358285175Writing region info on filesystem at 1731358285175Initializing all the Stores at 1731358285177 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285177Cleaning up temporary data from old regions at 1731358285185 (+8 ms)Running coprocessor post-open hooks at 1731358285195 (+10 ms)Region opened successfully at 1731358285195 2024-11-11T20:51:25,197 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708., pid=35, masterSystemTime=1731358285058 2024-11-11T20:51:25,197 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120., pid=47, masterSystemTime=1731358285154 2024-11-11T20:51:25,197 INFO [StoreOpener-bd0c630a3962dac6e1875877d78d1d43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN in 467 msec 2024-11-11T20:51:25,201 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:25,201 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:25,201 INFO [StoreOpener-bd0c630a3962dac6e1875877d78d1d43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd0c630a3962dac6e1875877d78d1d43 columnFamilyName family 2024-11-11T20:51:25,201 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:25,201 DEBUG [StoreOpener-bd0c630a3962dac6e1875877d78d1d43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,201 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,201 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:25,201 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:25,201 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a298d35593aa3585cf7f54bd6d37af4, NAME => 'test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-11T20:51:25,202 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:25,202 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 084ef38b701ab7eeaa42a673a92ec1d6, NAME => 'test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-11T20:51:25,202 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,202 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,202 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,202 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,203 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,203 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,203 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,203 INFO [StoreOpener-bd0c630a3962dac6e1875877d78d1d43-1 {}] regionserver.HStore(327): Store=bd0c630a3962dac6e1875877d78d1d43/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,203 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,203 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure bce7c26d0c06e82319102e4a44901ffa, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,206 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,206 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,206 INFO [StoreOpener-0a298d35593aa3585cf7f54bd6d37af4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,207 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,207 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,208 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:25,209 INFO [StoreOpener-0a298d35593aa3585cf7f54bd6d37af4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a298d35593aa3585cf7f54bd6d37af4 columnFamilyName family 2024-11-11T20:51:25,209 DEBUG [StoreOpener-0a298d35593aa3585cf7f54bd6d37af4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,210 INFO [StoreOpener-0a298d35593aa3585cf7f54bd6d37af4-1 {}] regionserver.HStore(327): Store=0a298d35593aa3585cf7f54bd6d37af4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,210 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,211 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,38917,1731358281807 because future has completed 2024-11-11T20:51:25,212 INFO [StoreOpener-084ef38b701ab7eeaa42a673a92ec1d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,214 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,215 INFO [StoreOpener-084ef38b701ab7eeaa42a673a92ec1d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 084ef38b701ab7eeaa42a673a92ec1d6 columnFamilyName family 2024-11-11T20:51:25,215 DEBUG [StoreOpener-084ef38b701ab7eeaa42a673a92ec1d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,215 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,216 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,216 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,216 INFO [StoreOpener-084ef38b701ab7eeaa42a673a92ec1d6-1 {}] regionserver.HStore(327): Store=084ef38b701ab7eeaa42a673a92ec1d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,217 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=20 2024-11-11T20:51:25,218 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure bce7c26d0c06e82319102e4a44901ffa, server=51ca66f7ee3c,36213,1731358281724 in 245 msec 2024-11-11T20:51:25,219 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,220 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,220 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,221 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bce7c26d0c06e82319102e4a44901ffa, ASSIGN in 491 msec 2024-11-11T20:51:25,223 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,227 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/bd0c630a3962dac6e1875877d78d1d43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,227 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/0a298d35593aa3585cf7f54bd6d37af4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,227 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/084ef38b701ab7eeaa42a673a92ec1d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,228 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 0a298d35593aa3585cf7f54bd6d37af4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68514469, jitterRate=0.020945146679878235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,228 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened bd0c630a3962dac6e1875877d78d1d43; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62284544, jitterRate=-0.07188796997070312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,228 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:25,228 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:25,229 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for bd0c630a3962dac6e1875877d78d1d43: Running coprocessor pre-open hook at 1731358285192Writing region info on filesystem at 1731358285192Initializing all the Stores at 1731358285194 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285194Cleaning up temporary data from old regions at 1731358285207 (+13 ms)Running coprocessor post-open hooks at 1731358285228 (+21 ms)Region opened successfully at 1731358285228 2024-11-11T20:51:25,229 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 0a298d35593aa3585cf7f54bd6d37af4: Running coprocessor pre-open hook at 1731358285203Writing region info on filesystem at 1731358285203Initializing all the Stores at 1731358285205 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285206 (+1 ms)Cleaning up temporary data from old regions at 1731358285216 (+10 ms)Running coprocessor post-open hooks at 1731358285228 (+12 ms)Region opened successfully at 1731358285228 2024-11-11T20:51:25,228 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 084ef38b701ab7eeaa42a673a92ec1d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67043687, jitterRate=-9.712129831314087E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,229 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:25,229 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 084ef38b701ab7eeaa42a673a92ec1d6: Running coprocessor pre-open hook at 1731358285203Writing region info on filesystem at 1731358285203Initializing all the Stores at 1731358285206 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285206Cleaning up temporary data from old regions at 1731358285220 (+14 ms)Running coprocessor post-open hooks at 1731358285229 (+9 ms)Region opened successfully at 1731358285229 2024-11-11T20:51:25,230 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43., pid=42, masterSystemTime=1731358285110 2024-11-11T20:51:25,230 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6., pid=34, masterSystemTime=1731358285058 2024-11-11T20:51:25,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=24 2024-11-11T20:51:25,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,38917,1731358281807 in 295 msec 2024-11-11T20:51:25,231 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4., pid=51, masterSystemTime=1731358285154 2024-11-11T20:51:25,233 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:25,233 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:25,234 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:25,234 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => f43990b9e210c622161ccc0992d6c911, NAME => 'test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-11T20:51:25,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN in 502 msec 2024-11-11T20:51:25,234 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,234 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,235 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,235 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,235 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:25,235 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:25,235 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e. 2024-11-11T20:51:25,235 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => f749f9c70b714192bc4197ca43f0d35e, NAME => 'test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-11T20:51:25,236 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,236 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,236 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,236 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,237 INFO [StoreOpener-f43990b9e210c622161ccc0992d6c911-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,238 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,239 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:25,240 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:25,240 INFO [StoreOpener-f43990b9e210c622161ccc0992d6c911-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f43990b9e210c622161ccc0992d6c911 columnFamilyName family 2024-11-11T20:51:25,240 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:25,240 DEBUG [StoreOpener-f43990b9e210c622161ccc0992d6c911-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,240 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => d8b6777ce113fcb1df22e25ffb49623a, NAME => 'test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-11T20:51:25,241 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,241 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,241 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,241 INFO [StoreOpener-f43990b9e210c622161ccc0992d6c911-1 {}] regionserver.HStore(327): Store=f43990b9e210c622161ccc0992d6c911/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,241 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,241 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd0c630a3962dac6e1875877d78d1d43, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,242 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,243 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,244 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,244 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,244 INFO [StoreOpener-f749f9c70b714192bc4197ca43f0d35e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,245 INFO [StoreOpener-d8b6777ce113fcb1df22e25ffb49623a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,248 INFO [StoreOpener-d8b6777ce113fcb1df22e25ffb49623a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8b6777ce113fcb1df22e25ffb49623a columnFamilyName family 2024-11-11T20:51:25,248 DEBUG [StoreOpener-d8b6777ce113fcb1df22e25ffb49623a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,248 INFO [StoreOpener-f749f9c70b714192bc4197ca43f0d35e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f749f9c70b714192bc4197ca43f0d35e columnFamilyName family 2024-11-11T20:51:25,248 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,248 DEBUG [StoreOpener-f749f9c70b714192bc4197ca43f0d35e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,249 INFO [StoreOpener-d8b6777ce113fcb1df22e25ffb49623a-1 {}] regionserver.HStore(327): Store=d8b6777ce113fcb1df22e25ffb49623a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,249 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,250 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,250 INFO [StoreOpener-f749f9c70b714192bc4197ca43f0d35e-1 {}] regionserver.HStore(327): Store=f749f9c70b714192bc4197ca43f0d35e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,250 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,252 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,252 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,252 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,252 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,253 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=29 2024-11-11T20:51:25,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure bd0c630a3962dac6e1875877d78d1d43, server=51ca66f7ee3c,36213,1731358281724 in 265 msec 2024-11-11T20:51:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:51:25,255 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,255 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,256 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/f43990b9e210c622161ccc0992d6c911/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,257 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,257 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened f43990b9e210c622161ccc0992d6c911; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59183668, jitterRate=-0.11809462308883667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,257 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:25,257 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for f43990b9e210c622161ccc0992d6c911: Running coprocessor pre-open hook at 1731358285235Writing region info on filesystem at 1731358285235Initializing all the Stores at 1731358285236 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285236Cleaning up temporary data from old regions at 1731358285244 (+8 ms)Running coprocessor post-open hooks at 1731358285257 (+13 ms)Region opened successfully at 1731358285257 2024-11-11T20:51:25,258 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bd0c630a3962dac6e1875877d78d1d43, ASSIGN in 527 msec 2024-11-11T20:51:25,260 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911., pid=41, masterSystemTime=1731358285110 2024-11-11T20:51:25,263 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,266 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d8b6777ce113fcb1df22e25ffb49623a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,267 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:25,267 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/f749f9c70b714192bc4197ca43f0d35e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a298d35593aa3585cf7f54bd6d37af4, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,267 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:25,268 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened d8b6777ce113fcb1df22e25ffb49623a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73470433, jitterRate=0.09479476511478424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,268 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:25,268 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:25,268 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for d8b6777ce113fcb1df22e25ffb49623a: Running coprocessor pre-open hook at 1731358285241Writing region info on filesystem at 1731358285241Initializing all the Stores at 1731358285243 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285243Cleaning up temporary data from old regions at 1731358285252 (+9 ms)Running coprocessor post-open hooks at 1731358285268 (+16 ms)Region opened successfully at 1731358285268 2024-11-11T20:51:25,268 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 43d3520c27ced34912ce528d771ce465, NAME => 'test,sss,1731358284091.43d3520c27ced34912ce528d771ce465.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-11T20:51:25,268 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened f749f9c70b714192bc4197ca43f0d35e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63581236, jitterRate=-0.05256575345993042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,268 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f749f9c70b714192bc4197ca43f0d35e 2024-11-11T20:51:25,268 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,268 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for f749f9c70b714192bc4197ca43f0d35e: Running coprocessor pre-open hook at 1731358285236Writing region info on filesystem at 1731358285236Initializing all the Stores at 1731358285238 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285238Cleaning up temporary data from old regions at 1731358285255 (+17 ms)Running coprocessor post-open hooks at 1731358285268 (+13 ms)Region opened successfully at 1731358285268 2024-11-11T20:51:25,269 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,269 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,sss,1731358284091.43d3520c27ced34912ce528d771ce465.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,269 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,269 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,271 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a., pid=48, masterSystemTime=1731358285154 2024-11-11T20:51:25,271 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e., pid=43, masterSystemTime=1731358285058 2024-11-11T20:51:25,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure f43990b9e210c622161ccc0992d6c911, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,276 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:25,276 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:25,277 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:25,277 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => eef5c8934a3876cee983f4197f34ce96, NAME => 'test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-11T20:51:25,278 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,278 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,278 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,278 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,281 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e. 2024-11-11T20:51:25,281 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,,1731358284091.f749f9c70b714192bc4197ca43f0d35e. 2024-11-11T20:51:25,281 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:25,281 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 885e30f0459dab7e5a3e27492bed3f2a, NAME => 'test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-11T20:51:25,282 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,282 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,282 INFO [StoreOpener-43d3520c27ced34912ce528d771ce465-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,282 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,282 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,283 INFO [StoreOpener-eef5c8934a3876cee983f4197f34ce96-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,285 INFO [StoreOpener-43d3520c27ced34912ce528d771ce465-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43d3520c27ced34912ce528d771ce465 columnFamilyName family 2024-11-11T20:51:25,285 DEBUG [StoreOpener-43d3520c27ced34912ce528d771ce465-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,285 INFO [StoreOpener-885e30f0459dab7e5a3e27492bed3f2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,286 INFO [StoreOpener-eef5c8934a3876cee983f4197f34ce96-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eef5c8934a3876cee983f4197f34ce96 columnFamilyName family 2024-11-11T20:51:25,286 INFO [StoreOpener-43d3520c27ced34912ce528d771ce465-1 {}] regionserver.HStore(327): Store=43d3520c27ced34912ce528d771ce465/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,286 DEBUG [StoreOpener-eef5c8934a3876cee983f4197f34ce96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,287 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=19 2024-11-11T20:51:25,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a298d35593aa3585cf7f54bd6d37af4, server=51ca66f7ee3c,36001,1731358281842 in 250 msec 2024-11-11T20:51:25,288 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,289 INFO [StoreOpener-885e30f0459dab7e5a3e27492bed3f2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 885e30f0459dab7e5a3e27492bed3f2a columnFamilyName family 2024-11-11T20:51:25,289 INFO [StoreOpener-eef5c8934a3876cee983f4197f34ce96-1 {}] regionserver.HStore(327): Store=eef5c8934a3876cee983f4197f34ce96/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,289 DEBUG [StoreOpener-885e30f0459dab7e5a3e27492bed3f2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,289 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,290 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,290 INFO [StoreOpener-885e30f0459dab7e5a3e27492bed3f2a-1 {}] regionserver.HStore(327): Store=885e30f0459dab7e5a3e27492bed3f2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,290 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,290 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,291 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,291 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,291 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,291 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,292 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,292 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,292 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0a298d35593aa3585cf7f54bd6d37af4, ASSIGN in 560 msec 2024-11-11T20:51:25,293 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,293 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f749f9c70b714192bc4197ca43f0d35e, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:25,295 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,295 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,296 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,38917,1731358281807 because future has completed 2024-11-11T20:51:25,300 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/885e30f0459dab7e5a3e27492bed3f2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,300 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 885e30f0459dab7e5a3e27492bed3f2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69122863, jitterRate=0.030010923743247986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,301 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:25,301 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 885e30f0459dab7e5a3e27492bed3f2a: Running coprocessor pre-open hook at 1731358285282Writing region info on filesystem at 1731358285282Initializing all the Stores at 1731358285284 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285284Cleaning up temporary data from old regions at 1731358285293 (+9 ms)Running coprocessor post-open hooks at 1731358285301 (+8 ms)Region opened successfully at 1731358285301 2024-11-11T20:51:25,302 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a., pid=33, masterSystemTime=1731358285058 2024-11-11T20:51:25,302 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/eef5c8934a3876cee983f4197f34ce96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,304 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened eef5c8934a3876cee983f4197f34ce96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74070488, jitterRate=0.1037362813949585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,305 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:25,305 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for eef5c8934a3876cee983f4197f34ce96: Running coprocessor pre-open hook at 1731358285278Writing region info on filesystem at 1731358285278Initializing all the Stores at 1731358285280 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285281 (+1 ms)Cleaning up temporary data from old regions at 1731358285292 (+11 ms)Running coprocessor post-open hooks at 1731358285305 (+13 ms)Region opened successfully at 1731358285305 2024-11-11T20:51:25,306 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/43d3520c27ced34912ce528d771ce465/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,306 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:25,306 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:25,306 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96., pid=52, masterSystemTime=1731358285154 2024-11-11T20:51:25,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=26 2024-11-11T20:51:25,307 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 43d3520c27ced34912ce528d771ce465; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63648752, jitterRate=-0.0515596866607666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure f43990b9e210c622161ccc0992d6c911, server=51ca66f7ee3c,36213,1731358281724 in 327 msec 2024-11-11T20:51:25,307 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:25,308 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 43d3520c27ced34912ce528d771ce465: Running coprocessor pre-open hook at 1731358285269Writing region info on filesystem at 1731358285269Initializing all the Stores at 1731358285274 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285274Cleaning up temporary data from old regions at 1731358285291 (+17 ms)Running coprocessor post-open hooks at 1731358285308 (+17 ms)Region opened successfully at 1731358285308 2024-11-11T20:51:25,309 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731358284091.43d3520c27ced34912ce528d771ce465., pid=38, masterSystemTime=1731358285110 2024-11-11T20:51:25,310 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:25,310 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:25,310 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:25,310 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b414e49a28b2cae7a6a20bc170cff4f, NAME => 'test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-11T20:51:25,311 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,311 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,311 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,311 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f43990b9e210c622161ccc0992d6c911, ASSIGN in 580 msec 2024-11-11T20:51:25,313 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:25,313 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:25,313 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:25,313 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => e6f7fe1f29c3816146fbdd9ef0338b39, NAME => 'test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-11T20:51:25,314 INFO [StoreOpener-0b414e49a28b2cae7a6a20bc170cff4f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,314 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=43d3520c27ced34912ce528d771ce465, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,314 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,314 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,314 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,314 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,317 INFO [StoreOpener-0b414e49a28b2cae7a6a20bc170cff4f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b414e49a28b2cae7a6a20bc170cff4f columnFamilyName family 2024-11-11T20:51:25,317 DEBUG [StoreOpener-0b414e49a28b2cae7a6a20bc170cff4f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43d3520c27ced34912ce528d771ce465, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,319 INFO [StoreOpener-0b414e49a28b2cae7a6a20bc170cff4f-1 {}] regionserver.HStore(327): Store=0b414e49a28b2cae7a6a20bc170cff4f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,319 INFO [StoreOpener-e6f7fe1f29c3816146fbdd9ef0338b39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,319 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,320 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,321 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,321 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,321 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,322 INFO [StoreOpener-e6f7fe1f29c3816146fbdd9ef0338b39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6f7fe1f29c3816146fbdd9ef0338b39 columnFamilyName family 2024-11-11T20:51:25,322 DEBUG [StoreOpener-e6f7fe1f29c3816146fbdd9ef0338b39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure eef5c8934a3876cee983f4197f34ce96, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,324 INFO [StoreOpener-e6f7fe1f29c3816146fbdd9ef0338b39-1 {}] regionserver.HStore(327): Store=e6f7fe1f29c3816146fbdd9ef0338b39/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,324 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,325 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,328 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,328 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/0b414e49a28b2cae7a6a20bc170cff4f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,329 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=23 2024-11-11T20:51:25,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 43d3520c27ced34912ce528d771ce465, server=51ca66f7ee3c,36213,1731358281724 in 371 msec 2024-11-11T20:51:25,329 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,329 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,329 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 0b414e49a28b2cae7a6a20bc170cff4f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66605039, jitterRate=-0.007507577538490295}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,329 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:25,330 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 0b414e49a28b2cae7a6a20bc170cff4f: Running coprocessor pre-open hook at 1731358285311Writing region info on filesystem at 1731358285311Initializing all the Stores at 1731358285313 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285313Cleaning up temporary data from old regions at 1731358285322 (+9 ms)Running coprocessor post-open hooks at 1731358285330 (+8 ms)Region opened successfully at 1731358285330 2024-11-11T20:51:25,331 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f., pid=56, masterSystemTime=1731358285154 2024-11-11T20:51:25,332 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=43d3520c27ced34912ce528d771ce465, ASSIGN in 601 msec 2024-11-11T20:51:25,334 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:25,334 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:25,334 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:25,335 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 57a71397de63e8cb71cb808d7888d245, NAME => 'test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-11T20:51:25,335 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,335 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,335 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,335 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,337 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/e6f7fe1f29c3816146fbdd9ef0338b39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,338 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened e6f7fe1f29c3816146fbdd9ef0338b39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71119501, jitterRate=0.05976314842700958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,338 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:25,338 INFO [StoreOpener-57a71397de63e8cb71cb808d7888d245-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,338 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for e6f7fe1f29c3816146fbdd9ef0338b39: Running coprocessor pre-open hook at 1731358285314Writing region info on filesystem at 1731358285314Initializing all the Stores at 1731358285318 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285318Cleaning up temporary data from old regions at 1731358285329 (+11 ms)Running coprocessor post-open hooks at 1731358285338 (+9 ms)Region opened successfully at 1731358285338 2024-11-11T20:51:25,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=8 2024-11-11T20:51:25,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure eef5c8934a3876cee983f4197f34ce96, server=51ca66f7ee3c,36001,1731358281842 in 298 msec 2024-11-11T20:51:25,340 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39., pid=39, masterSystemTime=1731358285110 2024-11-11T20:51:25,341 INFO [StoreOpener-57a71397de63e8cb71cb808d7888d245-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57a71397de63e8cb71cb808d7888d245 columnFamilyName family 2024-11-11T20:51:25,341 DEBUG [StoreOpener-57a71397de63e8cb71cb808d7888d245-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,342 INFO [StoreOpener-57a71397de63e8cb71cb808d7888d245-1 {}] regionserver.HStore(327): Store=57a71397de63e8cb71cb808d7888d245/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eef5c8934a3876cee983f4197f34ce96, ASSIGN in 612 msec 2024-11-11T20:51:25,342 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,343 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:25,343 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:25,343 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:25,343 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => caaf3dccceaaa47724f8b76041286a47, NAME => 'test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-11T20:51:25,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,344 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,344 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,344 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,344 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,344 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,345 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,347 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,347 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6f7fe1f29c3816146fbdd9ef0338b39, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,348 INFO [StoreOpener-caaf3dccceaaa47724f8b76041286a47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,353 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=test, region=e6f7fe1f29c3816146fbdd9ef0338b39. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T20:51:25,354 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b414e49a28b2cae7a6a20bc170cff4f, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,355 INFO [StoreOpener-caaf3dccceaaa47724f8b76041286a47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region caaf3dccceaaa47724f8b76041286a47 columnFamilyName family 2024-11-11T20:51:25,355 DEBUG [StoreOpener-caaf3dccceaaa47724f8b76041286a47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,355 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=51ca66f7ee3c,36001,1731358281842, table=test, region=0b414e49a28b2cae7a6a20bc170cff4f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T20:51:25,356 INFO [StoreOpener-caaf3dccceaaa47724f8b76041286a47-1 {}] regionserver.HStore(327): Store=caaf3dccceaaa47724f8b76041286a47/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,356 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,358 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,358 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/57a71397de63e8cb71cb808d7888d245/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,359 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,359 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 57a71397de63e8cb71cb808d7888d245; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68346853, jitterRate=0.018447473645210266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,359 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:25,360 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 57a71397de63e8cb71cb808d7888d245: Running coprocessor pre-open hook at 1731358285336Writing region info on filesystem at 1731358285336Initializing all the Stores at 1731358285337 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285337Cleaning up temporary data from old regions at 1731358285347 (+10 ms)Running coprocessor post-open hooks at 1731358285360 (+13 ms)Region opened successfully at 1731358285360 2024-11-11T20:51:25,361 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,361 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,361 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245., pid=53, masterSystemTime=1731358285154 2024-11-11T20:51:25,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=17 2024-11-11T20:51:25,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b414e49a28b2cae7a6a20bc170cff4f, server=51ca66f7ee3c,36001,1731358281842 in 301 msec 2024-11-11T20:51:25,364 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:25,364 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:25,364 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:25,365 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a56b479883cb03a610f18bcad1bfcc3, NAME => 'test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-11T20:51:25,365 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,365 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,365 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,365 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,365 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,368 INFO [StoreOpener-4a56b479883cb03a610f18bcad1bfcc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0b414e49a28b2cae7a6a20bc170cff4f, ASSIGN in 636 msec 2024-11-11T20:51:25,370 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,370 INFO [StoreOpener-4a56b479883cb03a610f18bcad1bfcc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a56b479883cb03a610f18bcad1bfcc3 columnFamilyName family 2024-11-11T20:51:25,371 DEBUG [StoreOpener-4a56b479883cb03a610f18bcad1bfcc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,371 INFO [StoreOpener-4a56b479883cb03a610f18bcad1bfcc3-1 {}] regionserver.HStore(327): Store=4a56b479883cb03a610f18bcad1bfcc3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,372 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,373 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a71397de63e8cb71cb808d7888d245, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,374 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,375 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,375 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,378 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=18 2024-11-11T20:51:25,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure e6f7fe1f29c3816146fbdd9ef0338b39, server=51ca66f7ee3c,36213,1731358281724 in 413 msec 2024-11-11T20:51:25,381 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/caaf3dccceaaa47724f8b76041286a47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,382 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/4a56b479883cb03a610f18bcad1bfcc3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,383 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 4a56b479883cb03a610f18bcad1bfcc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75348876, jitterRate=0.12278574705123901}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,383 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened caaf3dccceaaa47724f8b76041286a47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58948650, jitterRate=-0.12159666419029236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,383 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:25,383 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:25,383 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 4a56b479883cb03a610f18bcad1bfcc3: Running coprocessor pre-open hook at 1731358285366Writing region info on filesystem at 1731358285366Initializing all the Stores at 1731358285367 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285367Cleaning up temporary data from old regions at 1731358285375 (+8 ms)Running coprocessor post-open hooks at 1731358285383 (+8 ms)Region opened successfully at 1731358285383 2024-11-11T20:51:25,383 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for caaf3dccceaaa47724f8b76041286a47: Running coprocessor pre-open hook at 1731358285345Writing region info on filesystem at 1731358285345Initializing all the Stores at 1731358285346 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285346Cleaning up temporary data from old regions at 1731358285361 (+15 ms)Running coprocessor post-open hooks at 1731358285383 (+22 ms)Region opened successfully at 1731358285383 2024-11-11T20:51:25,384 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47., pid=49, masterSystemTime=1731358285110 2024-11-11T20:51:25,385 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3., pid=55, masterSystemTime=1731358285154 2024-11-11T20:51:25,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e6f7fe1f29c3816146fbdd9ef0338b39, ASSIGN in 651 msec 2024-11-11T20:51:25,386 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=51ca66f7ee3c,38917,1731358281807, table=test, region=f749f9c70b714192bc4197ca43f0d35e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T20:51:25,388 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:25,388 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:25,388 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:25,389 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => c574684bc1a84767063943424c84e3f2, NAME => 'test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-11T20:51:25,389 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,389 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,389 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,390 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,390 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:25,390 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:25,390 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:25,390 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => d0e0eaf6d4b3f10b135b25df3514191b, NAME => 'test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-11T20:51:25,391 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,391 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,391 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,391 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,392 INFO [StoreOpener-c574684bc1a84767063943424c84e3f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,394 INFO [StoreOpener-c574684bc1a84767063943424c84e3f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c574684bc1a84767063943424c84e3f2 columnFamilyName family 2024-11-11T20:51:25,394 DEBUG [StoreOpener-c574684bc1a84767063943424c84e3f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,394 INFO [StoreOpener-d0e0eaf6d4b3f10b135b25df3514191b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-11-11T20:51:25,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 57a71397de63e8cb71cb808d7888d245, server=51ca66f7ee3c,36001,1731358281842 in 344 msec 2024-11-11T20:51:25,395 INFO [StoreOpener-c574684bc1a84767063943424c84e3f2-1 {}] regionserver.HStore(327): Store=c574684bc1a84767063943424c84e3f2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,395 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,396 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,397 INFO [StoreOpener-d0e0eaf6d4b3f10b135b25df3514191b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0e0eaf6d4b3f10b135b25df3514191b columnFamilyName family 2024-11-11T20:51:25,397 DEBUG [StoreOpener-d0e0eaf6d4b3f10b135b25df3514191b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,397 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,398 INFO [StoreOpener-d0e0eaf6d4b3f10b135b25df3514191b-1 {}] regionserver.HStore(327): Store=d0e0eaf6d4b3f10b135b25df3514191b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,398 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,398 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=57a71397de63e8cb71cb808d7888d245, ASSIGN in 666 msec 2024-11-11T20:51:25,398 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,400 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,400 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,400 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,400 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,401 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,401 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a56b479883cb03a610f18bcad1bfcc3, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,405 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,405 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,405 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/c574684bc1a84767063943424c84e3f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,407 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened c574684bc1a84767063943424c84e3f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73202331, jitterRate=0.0907997339963913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,407 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:25,407 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for c574684bc1a84767063943424c84e3f2: Running coprocessor pre-open hook at 1731358285390Writing region info on filesystem at 1731358285390Initializing all the Stores at 1731358285391 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285391Cleaning up temporary data from old regions at 1731358285398 (+7 ms)Running coprocessor post-open hooks at 1731358285407 (+9 ms)Region opened successfully at 1731358285407 2024-11-11T20:51:25,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure caaf3dccceaaa47724f8b76041286a47, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,409 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2., pid=45, masterSystemTime=1731358285110 2024-11-11T20:51:25,412 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:25,412 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:25,412 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:25,412 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => fec0270188c1b9e90ffcce7addd0cf30, NAME => 'test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-11T20:51:25,413 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,413 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d0e0eaf6d4b3f10b135b25df3514191b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,413 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,413 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,413 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,414 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened d0e0eaf6d4b3f10b135b25df3514191b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68028669, jitterRate=0.013706162571907043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,414 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:25,414 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for d0e0eaf6d4b3f10b135b25df3514191b: Running coprocessor pre-open hook at 1731358285391Writing region info on filesystem at 1731358285391Initializing all the Stores at 1731358285394 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285394Cleaning up temporary data from old regions at 1731358285401 (+7 ms)Running coprocessor post-open hooks at 1731358285414 (+13 ms)Region opened successfully at 1731358285414 2024-11-11T20:51:25,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=14 2024-11-11T20:51:25,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a56b479883cb03a610f18bcad1bfcc3, server=51ca66f7ee3c,36001,1731358281842 in 357 msec 2024-11-11T20:51:25,415 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b., pid=46, masterSystemTime=1731358285154 2024-11-11T20:51:25,417 INFO [StoreOpener-fec0270188c1b9e90ffcce7addd0cf30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4a56b479883cb03a610f18bcad1bfcc3, ASSIGN in 687 msec 2024-11-11T20:51:25,418 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:25,418 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:25,419 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:25,419 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 9dfd755f05156eb757bfa3d4fdc2b62a, NAME => 'test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-11T20:51:25,419 INFO [StoreOpener-fec0270188c1b9e90ffcce7addd0cf30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fec0270188c1b9e90ffcce7addd0cf30 columnFamilyName family 2024-11-11T20:51:25,419 DEBUG [StoreOpener-fec0270188c1b9e90ffcce7addd0cf30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,420 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,420 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:25,420 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,420 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,421 INFO [StoreOpener-fec0270188c1b9e90ffcce7addd0cf30-1 {}] regionserver.HStore(327): Store=fec0270188c1b9e90ffcce7addd0cf30/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,421 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,423 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,423 INFO [StoreOpener-9dfd755f05156eb757bfa3d4fdc2b62a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0e0eaf6d4b3f10b135b25df3514191b, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,424 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=c574684bc1a84767063943424c84e3f2, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,425 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,425 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,426 INFO [StoreOpener-9dfd755f05156eb757bfa3d4fdc2b62a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9dfd755f05156eb757bfa3d4fdc2b62a columnFamilyName family 2024-11-11T20:51:25,426 DEBUG [StoreOpener-9dfd755f05156eb757bfa3d4fdc2b62a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:25,427 INFO [StoreOpener-9dfd755f05156eb757bfa3d4fdc2b62a-1 {}] regionserver.HStore(327): Store=9dfd755f05156eb757bfa3d4fdc2b62a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:25,428 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure c574684bc1a84767063943424c84e3f2, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,428 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,429 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,429 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,430 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,430 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,432 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/fec0270188c1b9e90ffcce7addd0cf30/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,433 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,434 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened fec0270188c1b9e90ffcce7addd0cf30; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70260447, jitterRate=0.04696224629878998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,434 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:25,434 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for fec0270188c1b9e90ffcce7addd0cf30: Running coprocessor pre-open hook at 1731358285414Writing region info on filesystem at 1731358285414Initializing all the Stores at 1731358285416 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285416Cleaning up temporary data from old regions at 1731358285425 (+9 ms)Running coprocessor post-open hooks at 1731358285434 (+9 ms)Region opened successfully at 1731358285434 2024-11-11T20:51:25,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=22 2024-11-11T20:51:25,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure d0e0eaf6d4b3f10b135b25df3514191b, server=51ca66f7ee3c,36001,1731358281842 in 429 msec 2024-11-11T20:51:25,436 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30., pid=50, masterSystemTime=1731358285110 2024-11-11T20:51:25,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d0e0eaf6d4b3f10b135b25df3514191b, ASSIGN in 707 msec 2024-11-11T20:51:25,439 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/9dfd755f05156eb757bfa3d4fdc2b62a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:51:25,439 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:25,439 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:25,440 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 9dfd755f05156eb757bfa3d4fdc2b62a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63730316, jitterRate=-0.05034428834915161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:25,440 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:25,440 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 9dfd755f05156eb757bfa3d4fdc2b62a: Running coprocessor pre-open hook at 1731358285420Writing region info on filesystem at 1731358285420Initializing all the Stores at 1731358285422 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358285423 (+1 ms)Cleaning up temporary data from old regions at 1731358285430 (+7 ms)Running coprocessor post-open hooks at 1731358285440 (+10 ms)Region opened successfully at 1731358285440 2024-11-11T20:51:25,441 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a., pid=54, masterSystemTime=1731358285154 2024-11-11T20:51:25,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=6 2024-11-11T20:51:25,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure c574684bc1a84767063943424c84e3f2, server=51ca66f7ee3c,36213,1731358281724 in 445 msec 2024-11-11T20:51:25,444 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:25,444 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:25,445 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9dfd755f05156eb757bfa3d4fdc2b62a, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c574684bc1a84767063943424c84e3f2, ASSIGN in 720 msec 2024-11-11T20:51:25,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=12 2024-11-11T20:51:25,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 9dfd755f05156eb757bfa3d4fdc2b62a, server=51ca66f7ee3c,36001,1731358281842 in 403 msec 2024-11-11T20:51:25,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9dfd755f05156eb757bfa3d4fdc2b62a, ASSIGN in 727 msec 2024-11-11T20:51:25,460 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=test, region=caaf3dccceaaa47724f8b76041286a47. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T20:51:25,461 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:25,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure fec0270188c1b9e90ffcce7addd0cf30, server=51ca66f7ee3c,36213,1731358281724 because future has completed 2024-11-11T20:51:25,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=9 2024-11-11T20:51:25,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure caaf3dccceaaa47724f8b76041286a47, server=51ca66f7ee3c,36213,1731358281724 in 448 msec 2024-11-11T20:51:25,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=caaf3dccceaaa47724f8b76041286a47, ASSIGN in 742 msec 2024-11-11T20:51:25,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=11 2024-11-11T20:51:25,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure fec0270188c1b9e90ffcce7addd0cf30, server=51ca66f7ee3c,36213,1731358281724 in 453 msec 2024-11-11T20:51:25,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fec0270188c1b9e90ffcce7addd0cf30, ASSIGN in 750 msec 2024-11-11T20:51:25,482 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:25,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,38917,1731358281807 because future has completed 2024-11-11T20:51:25,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=5 2024-11-11T20:51:25,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,38917,1731358281807 in 503 msec 2024-11-11T20:51:25,490 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=51ca66f7ee3c,38917,1731358281807, table=test, region=885e30f0459dab7e5a3e27492bed3f2a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-11T20:51:25,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN in 762 msec 2024-11-11T20:51:25,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-11T20:51:25,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,38917,1731358281807 in 576 msec 2024-11-11T20:51:25,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN in 768 msec 2024-11-11T20:51:25,500 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:25,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8b6777ce113fcb1df22e25ffb49623a, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,38917,1731358281807 because future has completed 2024-11-11T20:51:25,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=28 2024-11-11T20:51:25,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure d8b6777ce113fcb1df22e25ffb49623a, server=51ca66f7ee3c,36001,1731358281842 in 494 msec 2024-11-11T20:51:25,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d8b6777ce113fcb1df22e25ffb49623a, ASSIGN in 784 msec 2024-11-11T20:51:25,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=16 2024-11-11T20:51:25,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,38917,1731358281807 in 594 msec 2024-11-11T20:51:25,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN in 792 msec 2024-11-11T20:51:25,523 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:25,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5acc05cce6a62ba139f97712c529e120, server=51ca66f7ee3c,36001,1731358281842 because future has completed 2024-11-11T20:51:25,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=25 2024-11-11T20:51:25,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 5acc05cce6a62ba139f97712c529e120, server=51ca66f7ee3c,36001,1731358281842 in 521 msec 2024-11-11T20:51:25,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=4 2024-11-11T20:51:25,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5acc05cce6a62ba139f97712c529e120, ASSIGN in 804 msec 2024-11-11T20:51:25,538 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:51:25,538 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358285538"}]},"ts":"1731358285538"} 2024-11-11T20:51:25,541 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-11T20:51:25,543 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:51:25,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4440 sec 2024-11-11T20:51:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:51:26,265 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-11T20:51:26,265 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-11T20:51:26,266 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:51:26,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-11T20:51:26,287 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:51:26,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-11T20:51:26,289 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:51:26,296 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-11T20:51:26,323 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-11T20:51:26,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:51:26,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:51:26,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:51:26,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:51:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 57a71397de63e8cb71cb808d7888d245 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eef5c8934a3876cee983f4197f34ce96 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9dfd755f05156eb757bfa3d4fdc2b62a changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4a56b479883cb03a610f18bcad1bfcc3 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0b414e49a28b2cae7a6a20bc170cff4f changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0a298d35593aa3585cf7f54bd6d37af4 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d0e0eaf6d4b3f10b135b25df3514191b changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5acc05cce6a62ba139f97712c529e120 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d8b6777ce113fcb1df22e25ffb49623a changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c574684bc1a84767063943424c84e3f2 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region caaf3dccceaaa47724f8b76041286a47 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fec0270188c1b9e90ffcce7addd0cf30 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 96c53e851b8fac62f296d140948ca3ee changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e6f7fe1f29c3816146fbdd9ef0338b39 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bce7c26d0c06e82319102e4a44901ffa changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 43d3520c27ced34912ce528d771ce465 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f43990b9e210c622161ccc0992d6c911 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bd0c630a3962dac6e1875877d78d1d43 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f749f9c70b714192bc4197ca43f0d35e changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 48dede73c5017e00f25348e75653b620 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 885e30f0459dab7e5a3e27492bed3f2a changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 084ef38b701ab7eeaa42a673a92ec1d6 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1c7ea4c5037945c7e8566334a0de9a50 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 07a697e4da0c9ded22ab8014e5ed7708 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 201d6d0f92a6319d987b626f2b7bad8b changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d414ef171f23d90cb3b6da2b30e93f36 changed from -1.0 to 0.0, refreshing cache 2024-11-11T20:51:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-11T20:51:26,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-11T20:51:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-11T20:51:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:51:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-11T20:51:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-11T20:51:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-11T20:51:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:51:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:51:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-11T20:51:26,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:51:26,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:51:26,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-11T20:51:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-11T20:51:26,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-11T20:51:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-11T20:51:26,425 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:51:26,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-11T20:51:26,460 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=380, OpenFileDescriptor=647, MaxFileDescriptor=1048576, SystemLoadAverage=103, ProcessCount=11, AvailableMemoryMB=4457 2024-11-11T20:51:26,464 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-11T20:51:26,466 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-11T20:51:26,466 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:26,467 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 51ca66f7ee3c,38917,1731358281807: Simulated kill ***** 2024-11-11T20:51:26,467 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-11T20:51:26,470 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-11T20:51:26,473 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-11T20:51:26,474 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-11T20:51:26,474 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-11T20:51:26,474 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 309139184 }, "NonHeapMemoryUsage": { "committed": 151388160, "init": 7667712, "max": -1, "used": 149091656 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "51ca66f7ee3c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "51ca66f7ee3c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "51ca66f7ee3c" } ] } 2024-11-11T20:51:26,478 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38185 {}] master.MasterRpcServices(700): 51ca66f7ee3c,38917,1731358281807 reported a fatal error: ***** ABORTING region server 51ca66f7ee3c,38917,1731358281807: Simulated kill ***** 2024-11-11T20:51:26,480 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,38917,1731358281807' ***** 2024-11-11T20:51:26,480 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-11T20:51:26,481 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:51:26,481 INFO [RS:1;51ca66f7ee3c:38917 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-11T20:51:26,481 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:51:26,481 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:51:26,481 INFO [RS:1;51ca66f7ee3c:38917 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-11T20:51:26,481 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:51:26,482 INFO [RS:1;51ca66f7ee3c:38917 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;51ca66f7ee3c:38917. 2024-11-11T20:51:26,482 DEBUG [RS:1;51ca66f7ee3c:38917 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:51:26,488 DEBUG [RS:1;51ca66f7ee3c:38917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:26,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741836_1012 (size=32114) 2024-11-11T20:51:26,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741836_1012 (size=32114) 2024-11-11T20:51:26,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741836_1012 (size=32114) 2024-11-11T20:51:26,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741834_1010 (size=2232) 2024-11-11T20:51:26,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741834_1010 (size=2232) 2024-11-11T20:51:26,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741834_1010 (size=2232) 2024-11-11T20:51:26,502 DEBUG [RS:1;51ca66f7ee3c:38917 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:26,503 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:51:26,503 INFO [RS:1;51ca66f7ee3c:38917 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38917 2024-11-11T20:51:26,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,38917,1731358281807 2024-11-11T20:51:26,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:51:26,511 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:51:26,514 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,38917,1731358281807] 2024-11-11T20:51:26,515 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,38917,1731358281807 already deleted, retry=false 2024-11-11T20:51:26,516 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 51ca66f7ee3c,38917,1731358281807 on 51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:26,522 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:26,527 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 51ca66f7ee3c,38917,1731358281807 (carryingMeta=true) 51ca66f7ee3c,38917,1731358281807/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@65c3278d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-11T20:51:26,528 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:26,530 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true, isMeta: true 2024-11-11T20:51:26,532 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting 2024-11-11T20:51:26,534 INFO [PEWorker-1 {}] master.SplitWALManager(105): 51ca66f7ee3c,38917,1731358281807 WAL count=1, meta=true 2024-11-11T20:51:26,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta}] 2024-11-11T20:51:26,543 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:26,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:26,573 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:51:26,583 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T20:51:26,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:51:26,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38917-0x100308e27080002, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:51:26,615 INFO [RS:1;51ca66f7ee3c:38917 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:51:26,615 INFO [RS:1;51ca66f7ee3c:38917 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,38917,1731358281807; zookeeper connection closed. 2024-11-11T20:51:26,615 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ea152d5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ea152d5 2024-11-11T20:51:26,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-11T20:51:26,735 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, size=31.4 K (32114bytes) 2024-11-11T20:51:26,736 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-11T20:51:26,736 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-11T20:51:26,736 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta 2024-11-11T20:51:26,741 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta after 4ms 2024-11-11T20:51:26,748 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta took 14ms 2024-11-11T20:51:26,759 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-11T20:51:26,765 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta.temp 2024-11-11T20:51:26,769 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta.temp 2024-11-11T20:51:26,772 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta so closing down 2024-11-11T20:51:26,772 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-11T20:51:26,774 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-11T20:51:26,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741864_1040 (size=31873) 2024-11-11T20:51:26,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741864_1040 (size=31873) 2024-11-11T20:51:26,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741864_1040 (size=31873) 2024-11-11T20:51:26,785 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-11T20:51:26,787 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta.temp to hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-11T20:51:26,790 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-11T20:51:26,791 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, journal: Splitting hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, size=31.4 K (32114bytes) at 1731358286735Creating recovered edits writer path=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta.temp at 1731358286769 (+34 ms)Finishing writing output for hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta so closing down at 1731358286772 (+3 ms)3 split writer threads finished at 1731358286774 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1731358286785 (+11 ms)Rename recovered edits hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta.temp to hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731358286787 (+2 ms)Processed 58 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731358286791 (+4 ms) 2024-11-11T20:51:26,791 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta 2024-11-11T20:51:26,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-11T20:51:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-11T20:51:26,802 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta to hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs 2024-11-11T20:51:26,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T20:51:26,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 in 257 msec 2024-11-11T20:51:26,807 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:26,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-11T20:51:26,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 in 272 msec 2024-11-11T20:51:26,813 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting dir is empty, no logs to split. 2024-11-11T20:51:26,813 INFO [PEWorker-1 {}] master.SplitWALManager(105): 51ca66f7ee3c,38917,1731358281807 WAL count=0, meta=true 2024-11-11T20:51:26,814 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 51ca66f7ee3c,38917,1731358281807 WAL splitting is done? wals=0, meta=true 2024-11-11T20:51:26,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T20:51:26,817 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:26,818 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:26,967 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T20:51:26,967 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T20:51:26,968 DEBUG [51ca66f7ee3c:38185 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-11T20:51:26,969 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:51:26,969 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:51:26,970 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:51:26,970 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:51:26,970 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:51:26,970 INFO [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:51:26,970 INFO [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:51:26,970 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:51:26,970 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:26,973 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPENING 2024-11-11T20:51:26,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:26,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:26,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:26,975 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:26,976 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:26,976 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:26,976 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:26,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:27,136 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T20:51:27,136 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:51:27,137 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T20:51:27,140 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C36213%2C1731358281724.meta, suffix=.meta, logDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36213,1731358281724, archiveDir=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs, maxLogs=32 2024-11-11T20:51:27,158 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36213,1731358281724/51ca66f7ee3c%2C36213%2C1731358281724.meta.1731358287143.meta, exclude list is [], retry=0 2024-11-11T20:51:27,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36815,DS-7b35a41a-1c7b-4e15-8fdf-7861a2284fc2,DISK] 2024-11-11T20:51:27,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45587,DS-b1845f9b-ff07-4be7-bc46-1cdf58d7c517,DISK] 2024-11-11T20:51:27,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38355,DS-149c8a03-8d45-412c-8335-e18d2a7471f3,DISK] 2024-11-11T20:51:27,164 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36213,1731358281724/51ca66f7ee3c%2C36213%2C1731358281724.meta.1731358287143.meta 2024-11-11T20:51:27,165 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35225:35225),(127.0.0.1/127.0.0.1:37191:37191),(127.0.0.1/127.0.0.1:41935:41935)] 2024-11-11T20:51:27,165 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:51:27,166 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T20:51:27,166 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T20:51:27,166 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T20:51:27,166 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T20:51:27,166 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:27,166 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T20:51:27,167 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T20:51:27,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T20:51:27,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:51:27,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:27,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:27,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T20:51:27,173 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:51:27,173 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:27,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:27,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T20:51:27,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:51:27,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:27,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:27,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T20:51:27,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:51:27,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:27,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:51:27,179 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:51:27,180 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740 2024-11-11T20:51:27,183 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740 2024-11-11T20:51:27,184 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-11T20:51:27,203 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-11T20:51:27,207 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-11T20:51:27,285 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/info/54fb1d5425fe4deda47f16bee111db0a is 138, key is test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2./info:regioninfo/1731358285424/Put/seqid=0 2024-11-11T20:51:27,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741866_1042 (size=34924) 2024-11-11T20:51:27,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741866_1042 (size=34924) 2024-11-11T20:51:27,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741866_1042 (size=34924) 2024-11-11T20:51:27,299 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/info/54fb1d5425fe4deda47f16bee111db0a 2024-11-11T20:51:27,376 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/ns/4495084f0bb5497eaeb5e83d4288c5fc is 43, key is default/ns:d/1731358283783/Put/seqid=0 2024-11-11T20:51:27,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741867_1043 (size=5153) 2024-11-11T20:51:27,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741867_1043 (size=5153) 2024-11-11T20:51:27,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741867_1043 (size=5153) 2024-11-11T20:51:27,385 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/ns/4495084f0bb5497eaeb5e83d4288c5fc 2024-11-11T20:51:27,412 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/table/47eee235ee794d56b271b15d89794d2b is 40, key is test/table:state/1731358285538/Put/seqid=0 2024-11-11T20:51:27,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741868_1044 (size=5165) 2024-11-11T20:51:27,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741868_1044 (size=5165) 2024-11-11T20:51:27,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741868_1044 (size=5165) 2024-11-11T20:51:27,424 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/table/47eee235ee794d56b271b15d89794d2b 2024-11-11T20:51:27,434 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/info/54fb1d5425fe4deda47f16bee111db0a as hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/info/54fb1d5425fe4deda47f16bee111db0a 2024-11-11T20:51:27,445 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/info/54fb1d5425fe4deda47f16bee111db0a, entries=260, sequenceid=60, filesize=34.1 K 2024-11-11T20:51:27,448 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/ns/4495084f0bb5497eaeb5e83d4288c5fc as hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/ns/4495084f0bb5497eaeb5e83d4288c5fc 2024-11-11T20:51:27,458 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/ns/4495084f0bb5497eaeb5e83d4288c5fc, entries=2, sequenceid=60, filesize=5.0 K 2024-11-11T20:51:27,460 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/table/47eee235ee794d56b271b15d89794d2b as hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/table/47eee235ee794d56b271b15d89794d2b 2024-11-11T20:51:27,467 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/table/47eee235ee794d56b271b15d89794d2b, entries=2, sequenceid=60, filesize=5.0 K 2024-11-11T20:51:27,471 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 264ms, sequenceid=60, compaction requested=false; wal=null 2024-11-11T20:51:27,472 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T20:51:27,475 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-11T20:51:27,477 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:51:27,477 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:51:27,478 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:51:27,479 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:51:27,483 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-11T20:51:27,484 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74649989, jitterRate=0.1123715192079544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:51:27,484 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T20:51:27,484 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731358287167Writing region info on filesystem at 1731358287167Initializing all the Stores at 1731358287168 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358287168Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358287169 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358287169Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358287169Obtaining lock to block concurrent updates at 1731358287208 (+39 ms)Preparing flush snapshotting stores in 1588230740 at 1731358287208Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731358287211 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731358287212 (+1 ms)Flushing 1588230740/info: creating writer at 1731358287214 (+2 ms)Flushing 1588230740/info: appending metadata at 1731358287274 (+60 ms)Flushing 1588230740/info: closing flushed file at 1731358287276 (+2 ms)Flushing 1588230740/ns: creating writer at 1731358287353 (+77 ms)Flushing 1588230740/ns: appending metadata at 1731358287374 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731358287374Flushing 1588230740/table: creating writer at 1731358287395 (+21 ms)Flushing 1588230740/table: appending metadata at 1731358287411 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731358287411Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@432fc4a9: reopening flushed file at 1731358287433 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1286c59d: reopening flushed file at 1731358287447 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5691c605: reopening flushed file at 1731358287459 (+12 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 264ms, sequenceid=60, compaction requested=false; wal=null at 1731358287471 (+12 ms)Cleaning up temporary data from old regions at 1731358287477 (+6 ms)Running coprocessor post-open hooks at 1731358287484 (+7 ms)Region opened successfully at 1731358287484 2024-11-11T20:51:27,485 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731358287130 2024-11-11T20:51:27,488 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T20:51:27,488 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T20:51:27,489 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:27,490 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:27,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:27,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:27,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:27,491 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:27,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:27,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T20:51:27,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 in 515 msec 2024-11-11T20:51:27,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-11T20:51:27,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 679 msec 2024-11-11T20:51:27,498 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 51ca66f7ee3c,38917,1731358281807 had 9 regions 2024-11-11T20:51:27,499 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true, isMeta: false 2024-11-11T20:51:27,502 INFO [PEWorker-1 {}] master.SplitWALManager(105): 51ca66f7ee3c,38917,1731358281807 WAL count=1, meta=false 2024-11-11T20:51:27,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129}] 2024-11-11T20:51:27,505 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:27,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:27,505 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:27,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:27,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36213 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-11T20:51:27,692 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, size=2.2 K (2232bytes) 2024-11-11T20:51:27,692 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129 2024-11-11T20:51:27,693 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129 after 1ms 2024-11-11T20:51:27,696 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129 took 5ms 2024-11-11T20:51:27,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d414ef171f23d90cb3b6da2b30e93f36: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T20:51:27,702 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 48dede73c5017e00f25348e75653b620: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T20:51:27,704 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1c7ea4c5037945c7e8566334a0de9a50: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T20:51:27,706 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 201d6d0f92a6319d987b626f2b7bad8b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T20:51:27,708 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 07a697e4da0c9ded22ab8014e5ed7708: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T20:51:27,710 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 084ef38b701ab7eeaa42a673a92ec1d6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T20:51:27,713 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f749f9c70b714192bc4197ca43f0d35e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T20:51:27,715 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 885e30f0459dab7e5a3e27492bed3f2a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-11T20:51:27,715 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129 so closing down 2024-11-11T20:51:27,715 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-11T20:51:27,715 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-11T20:51:27,715 INFO [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-11-11T20:51:27,715 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, journal: Splitting hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, size=2.2 K (2232bytes) at 1731358287692Finishing writing output for hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129 so closing down at 1731358287715 (+23 ms)3 split writer threads finished at 1731358287715Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1731358287715 2024-11-11T20:51:27,715 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129 2024-11-11T20:51:27,715 DEBUG [RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-11T20:51:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38185 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-11T20:51:27,720 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting/51ca66f7ee3c%2C38917%2C1731358281807.1731358283129 to hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs 2024-11-11T20:51:27,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T20:51:27,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 in 214 msec 2024-11-11T20:51:27,726 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:27,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-11T20:51:27,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 in 224 msec 2024-11-11T20:51:27,731 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,38917,1731358281807-splitting dir is empty, no logs to split. 2024-11-11T20:51:27,731 INFO [PEWorker-1 {}] master.SplitWALManager(105): 51ca66f7ee3c,38917,1731358281807 WAL count=0, meta=false 2024-11-11T20:51:27,732 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 51ca66f7ee3c,38917,1731358281807 WAL splitting is done? wals=0, meta=false 2024-11-11T20:51:27,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN}] 2024-11-11T20:51:27,737 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:27,737 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:27,738 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:27,739 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:27,740 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:27,742 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:27,743 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:27,744 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:27,745 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:27,746 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:27,747 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:27,748 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:27,749 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:27,750 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:27,751 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:27,752 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T20:51:27,888 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(204): Hosts are {51ca66f7ee3c=0} racks are {/default-rack=0} 2024-11-11T20:51:27,888 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T20:51:27,889 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T20:51:27,889 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T20:51:27,889 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T20:51:27,889 INFO [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T20:51:27,889 INFO [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T20:51:27,889 DEBUG [51ca66f7ee3c:38185 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T20:51:27,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:27,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:27,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:27,890 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 51ca66f7ee3c:38917 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:51:27,891 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, regionLocation=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:27,891 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:27,892 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:27,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1, error=java.net.ConnectException: Call to address=51ca66f7ee3c:38917 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,892 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:27,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-11T20:51:27,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 from cache 2024-11-11T20:51:27,893 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=48dede73c5017e00f25348e75653b620, regionState=OPENING, regionLocation=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:27,893 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:51:27,893 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,36213,1731358281724, seqNum=-1] 2024-11-11T20:51:27,894 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:51:27,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 51ca66f7ee3c:38917 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,895 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 51ca66f7ee3c:38917 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:51:27,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,36213,1731358281724, seqNum=-1, error=java.net.ConnectException: Call to address=51ca66f7ee3c:38917 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,36213,1731358281724, seqNum=-1, error=java.net.ConnectException: Call to address=51ca66f7ee3c:38917 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,36213,1731358281724, seqNum=-1, error=java.net.ConnectException: Call to address=51ca66f7ee3c:38917 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47451, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:51:27,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,36213,1731358281724, seqNum=-1, error=java.net.ConnectException: Call to address=51ca66f7ee3c:38917 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,36213,1731358281724, seqNum=-1, error=java.net.ConnectException: Call to address=51ca66f7ee3c:38917 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,38917,1731358281807, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,36213,1731358281724, seqNum=-1, error=java.net.ConnectException: Call to address=51ca66f7ee3c:38917 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 51ca66f7ee3c:38917 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 51ca66f7ee3c/172.17.0.2:38917 2024-11-11T20:51:27,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN because future has completed 2024-11-11T20:51:27,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:28,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN because future has completed 2024-11-11T20:51:28,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:28,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN because future has completed 2024-11-11T20:51:28,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN because future has completed 2024-11-11T20:51:28,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN because future has completed 2024-11-11T20:51:28,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN because future has completed 2024-11-11T20:51:28,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:28,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN because future has completed 2024-11-11T20:51:28,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN because future has completed 2024-11-11T20:51:28,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:28,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842}] 2024-11-11T20:51:28,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:28,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:28,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7ea4c5037945c7e8566334a0de9a50, server=51ca66f7ee3c,36213,1731358281724}] 2024-11-11T20:51:28,057 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:28,057 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 084ef38b701ab7eeaa42a673a92ec1d6, NAME => 'test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-11T20:51:28,058 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,058 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:28,058 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,058 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,060 INFO [StoreOpener-084ef38b701ab7eeaa42a673a92ec1d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,061 INFO [StoreOpener-084ef38b701ab7eeaa42a673a92ec1d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 084ef38b701ab7eeaa42a673a92ec1d6 columnFamilyName family 2024-11-11T20:51:28,061 DEBUG [StoreOpener-084ef38b701ab7eeaa42a673a92ec1d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:28,062 INFO [StoreOpener-084ef38b701ab7eeaa42a673a92ec1d6-1 {}] regionserver.HStore(327): Store=084ef38b701ab7eeaa42a673a92ec1d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:28,062 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,063 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,064 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,065 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,065 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,068 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,069 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 084ef38b701ab7eeaa42a673a92ec1d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60577592, jitterRate=-0.09732353687286377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:28,069 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,070 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 084ef38b701ab7eeaa42a673a92ec1d6: Running coprocessor pre-open hook at 1731358288058Writing region info on filesystem at 1731358288058Initializing all the Stores at 1731358288059 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358288059Cleaning up temporary data from old regions at 1731358288065 (+6 ms)Running coprocessor post-open hooks at 1731358288069 (+4 ms)Region opened successfully at 1731358288069 2024-11-11T20:51:28,071 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6., pid=75, masterSystemTime=1731358288052 2024-11-11T20:51:28,073 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:28,074 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:28,074 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:28,074 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 48dede73c5017e00f25348e75653b620, NAME => 'test,fff,1731358284091.48dede73c5017e00f25348e75653b620.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-11T20:51:28,075 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,075 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,fff,1731358284091.48dede73c5017e00f25348e75653b620.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:28,075 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,075 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,076 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-11T20:51:28,076 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,077 INFO [StoreOpener-48dede73c5017e00f25348e75653b620-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,077 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,078 INFO [StoreOpener-48dede73c5017e00f25348e75653b620-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48dede73c5017e00f25348e75653b620 columnFamilyName family 2024-11-11T20:51:28,078 DEBUG [StoreOpener-48dede73c5017e00f25348e75653b620-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:28,079 INFO [StoreOpener-48dede73c5017e00f25348e75653b620-1 {}] regionserver.HStore(327): Store=48dede73c5017e00f25348e75653b620/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:28,079 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,080 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,082 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,083 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,083 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,086 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,087 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 48dede73c5017e00f25348e75653b620; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70123617, jitterRate=0.0449233204126358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:28,087 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,087 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 48dede73c5017e00f25348e75653b620: Running coprocessor pre-open hook at 1731358288075Writing region info on filesystem at 1731358288075Initializing all the Stores at 1731358288077 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358288077Cleaning up temporary data from old regions at 1731358288083 (+6 ms)Running coprocessor post-open hooks at 1731358288087 (+4 ms)Region opened successfully at 1731358288087 2024-11-11T20:51:28,088 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731358284091.48dede73c5017e00f25348e75653b620., pid=72, masterSystemTime=1731358288052 2024-11-11T20:51:28,087 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-11T20:51:28,090 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,090 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7ea4c5037945c7e8566334a0de9a50, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,091 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7ea4c5037945c7e8566334a0de9a50, server=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,091 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:28,091 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:28,091 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:28,092 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => d414ef171f23d90cb3b6da2b30e93f36, NAME => 'test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-11T20:51:28,092 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,092 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:28,092 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,092 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,094 INFO [StoreOpener-d414ef171f23d90cb3b6da2b30e93f36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,095 INFO [StoreOpener-d414ef171f23d90cb3b6da2b30e93f36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d414ef171f23d90cb3b6da2b30e93f36 columnFamilyName family 2024-11-11T20:51:28,096 DEBUG [StoreOpener-d414ef171f23d90cb3b6da2b30e93f36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:28,096 INFO [StoreOpener-d414ef171f23d90cb3b6da2b30e93f36-1 {}] regionserver.HStore(327): Store=d414ef171f23d90cb3b6da2b30e93f36/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:28,096 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,097 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,099 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,099 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,099 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,101 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,102 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened d414ef171f23d90cb3b6da2b30e93f36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68106413, jitterRate=0.014864638447761536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:28,102 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,103 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for d414ef171f23d90cb3b6da2b30e93f36: Running coprocessor pre-open hook at 1731358288093Writing region info on filesystem at 1731358288093Initializing all the Stores at 1731358288094 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358288094Cleaning up temporary data from old regions at 1731358288099 (+5 ms)Running coprocessor post-open hooks at 1731358288102 (+3 ms)Region opened successfully at 1731358288102 2024-11-11T20:51:28,104 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36., pid=74, masterSystemTime=1731358288052 2024-11-11T20:51:28,106 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:28,106 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:28,106 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:28,107 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 07a697e4da0c9ded22ab8014e5ed7708, NAME => 'test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-11T20:51:28,107 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,107 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:28,107 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,107 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,109 INFO [StoreOpener-07a697e4da0c9ded22ab8014e5ed7708-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,110 INFO [StoreOpener-07a697e4da0c9ded22ab8014e5ed7708-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07a697e4da0c9ded22ab8014e5ed7708 columnFamilyName family 2024-11-11T20:51:28,110 DEBUG [StoreOpener-07a697e4da0c9ded22ab8014e5ed7708-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:28,111 INFO [StoreOpener-07a697e4da0c9ded22ab8014e5ed7708-1 {}] regionserver.HStore(327): Store=07a697e4da0c9ded22ab8014e5ed7708/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:28,111 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,113 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,115 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,116 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,116 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,118 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,119 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 07a697e4da0c9ded22ab8014e5ed7708; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65781090, jitterRate=-0.019785374402999878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:28,119 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,120 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 07a697e4da0c9ded22ab8014e5ed7708: Running coprocessor pre-open hook at 1731358288107Writing region info on filesystem at 1731358288107Initializing all the Stores at 1731358288108 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358288109 (+1 ms)Cleaning up temporary data from old regions at 1731358288116 (+7 ms)Running coprocessor post-open hooks at 1731358288119 (+3 ms)Region opened successfully at 1731358288120 (+1 ms) 2024-11-11T20:51:28,121 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708., pid=76, masterSystemTime=1731358288052 2024-11-11T20:51:28,124 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:28,124 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:28,164 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:28,164 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 885e30f0459dab7e5a3e27492bed3f2a, NAME => 'test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-11T20:51:28,165 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,165 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:28,165 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,165 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,167 INFO [StoreOpener-885e30f0459dab7e5a3e27492bed3f2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,169 INFO [StoreOpener-885e30f0459dab7e5a3e27492bed3f2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 885e30f0459dab7e5a3e27492bed3f2a columnFamilyName family 2024-11-11T20:51:28,169 DEBUG [StoreOpener-885e30f0459dab7e5a3e27492bed3f2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:28,169 INFO [StoreOpener-885e30f0459dab7e5a3e27492bed3f2a-1 {}] regionserver.HStore(327): Store=885e30f0459dab7e5a3e27492bed3f2a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:28,170 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,170 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,172 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,172 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,172 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,174 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,175 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 885e30f0459dab7e5a3e27492bed3f2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65392921, jitterRate=-0.02556954324245453}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:28,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,175 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 885e30f0459dab7e5a3e27492bed3f2a: Running coprocessor pre-open hook at 1731358288165Writing region info on filesystem at 1731358288165Initializing all the Stores at 1731358288167 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358288167Cleaning up temporary data from old regions at 1731358288172 (+5 ms)Running coprocessor post-open hooks at 1731358288175 (+3 ms)Region opened successfully at 1731358288175 2024-11-11T20:51:28,176 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a., pid=78, masterSystemTime=1731358288160 2024-11-11T20:51:28,179 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:28,179 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:28,179 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. 2024-11-11T20:51:28,179 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c7ea4c5037945c7e8566334a0de9a50, NAME => 'test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-11T20:51:28,180 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,180 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:51:28,180 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,180 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,181 INFO [StoreOpener-1c7ea4c5037945c7e8566334a0de9a50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,183 INFO [StoreOpener-1c7ea4c5037945c7e8566334a0de9a50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c7ea4c5037945c7e8566334a0de9a50 columnFamilyName family 2024-11-11T20:51:28,183 DEBUG [StoreOpener-1c7ea4c5037945c7e8566334a0de9a50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:51:28,183 INFO [StoreOpener-1c7ea4c5037945c7e8566334a0de9a50-1 {}] regionserver.HStore(327): Store=1c7ea4c5037945c7e8566334a0de9a50/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:51:28,183 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,184 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,185 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,186 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,186 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,187 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,187 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,187 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,187 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,188 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,188 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,188 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,189 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,189 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 1c7ea4c5037945c7e8566334a0de9a50; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72829719, jitterRate=0.08524738252162933}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T20:51:28,189 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c7ea4c5037945c7e8566334a0de9a50 2024-11-11T20:51:28,189 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 1c7ea4c5037945c7e8566334a0de9a50: Running coprocessor pre-open hook at 1731358288180Writing region info on filesystem at 1731358288180Initializing all the Stores at 1731358288181 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358288181Cleaning up temporary data from old regions at 1731358288186 (+5 ms)Running coprocessor post-open hooks at 1731358288189 (+3 ms)Region opened successfully at 1731358288189 2024-11-11T20:51:28,189 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,190 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50., pid=79, masterSystemTime=1731358288160 2024-11-11T20:51:28,191 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,198 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,203 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7ea4c5037945c7e8566334a0de9a50, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,210 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,210 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,212 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,212 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,213 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,213 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,213 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,213 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c7ea4c5037945c7e8566334a0de9a50, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 23 msec 2024-11-11T20:51:28,216 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,216 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,216 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,217 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,218 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,218 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,217 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "51ca66f7ee3c" port: 36213 start_code: 1731358281724 } transition { transition_code: OPENED region_info { region_id: 1731358284091 table_name { namespace: "default" qualifier: "test" } start_key: "qqq" end_key: "rrr" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1731358281918 }; retry (#0) after 1005ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-11T20:51:28,218 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,221 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318221 2024-11-11T20:51:28,221 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,221 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,223 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,229 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,250 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,256 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,256 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,256 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348256 2024-11-11T20:51:28,256 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408256 2024-11-11T20:51:28,256 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 35 msec 2024-11-11T20:51:28,256 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,256 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,256 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,257 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,257 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,257 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,258 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,265 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,266 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,266 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,272 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,273 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,274 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,274 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,275 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 885e30f0459dab7e5a3e27492bed3f2a, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,275 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-11T20:51:28,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,277 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,277 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,278 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,278 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,278 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,279 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,280 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318280 2024-11-11T20:51:28,280 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,280 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,281 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,282 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,282 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,283 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,290 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,291 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,292 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,293 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,293 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,293 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348293 2024-11-11T20:51:28,293 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408293 2024-11-11T20:51:28,294 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-11T20:51:28,294 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,294 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,294 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,294 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,294 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,294 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,294 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,294 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,299 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,300 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,301 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,304 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,305 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,305 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,305 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,305 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,305 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,305 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,306 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,306 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,306 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,306 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,306 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,306 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure f749f9c70b714192bc4197ca43f0d35e, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,306 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-11T20:51:28,307 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,307 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,308 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,311 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318311 2024-11-11T20:51:28,311 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,311 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,313 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,314 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,314 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,327 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,327 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,327 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348327 2024-11-11T20:51:28,327 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408327 2024-11-11T20:51:28,327 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-11T20:51:28,327 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,327 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,328 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,328 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,328 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,328 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,329 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,334 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,334 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,334 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,334 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,334 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,334 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,334 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,337 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,342 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,342 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,342 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,342 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,342 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,342 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,342 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,343 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,343 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,343 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,343 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,343 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,343 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,343 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,343 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07a697e4da0c9ded22ab8014e5ed7708, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,343 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-11T20:51:28,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,345 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,345 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,347 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,347 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,347 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,348 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,348 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,348 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,348 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,348 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,348 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318348 2024-11-11T20:51:28,348 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,348 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,349 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,351 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,351 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,351 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,359 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,359 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,359 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348359 2024-11-11T20:51:28,359 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408359 2024-11-11T20:51:28,359 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-11T20:51:28,359 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,360 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,360 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,360 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,360 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,360 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,361 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,364 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,366 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,368 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,369 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,369 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,369 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,370 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,370 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,370 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-11T20:51:28,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,371 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,371 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,373 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,374 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,375 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318375 2024-11-11T20:51:28,375 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,375 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,376 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,377 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,388 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,388 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348388 2024-11-11T20:51:28,388 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408388 2024-11-11T20:51:28,388 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-11T20:51:28,388 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,389 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,389 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,389 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,389 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,389 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,389 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,389 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,390 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,395 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,400 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,400 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,400 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,400 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure d414ef171f23d90cb3b6da2b30e93f36, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-11T20:51:28,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,402 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,402 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,404 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,404 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,404 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,404 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,404 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,405 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,405 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318405 2024-11-11T20:51:28,405 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,405 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,406 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,407 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,408 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,408 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,417 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,417 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,417 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348417 2024-11-11T20:51:28,417 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408417 2024-11-11T20:51:28,417 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-11T20:51:28,417 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,417 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,417 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,417 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,417 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,417 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,417 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,418 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,418 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,420 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,421 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,422 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,422 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,424 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,425 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,426 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,426 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,426 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,426 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,426 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,426 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,426 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,426 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,426 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,427 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,427 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,427 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 201d6d0f92a6319d987b626f2b7bad8b, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,427 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-11T20:51:28,428 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,428 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,428 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,430 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,431 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,432 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318432 2024-11-11T20:51:28,432 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,432 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,434 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,437 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,438 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,438 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,442 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,442 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348442 2024-11-11T20:51:28,442 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408442 2024-11-11T20:51:28,442 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-11T20:51:28,442 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,442 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,442 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,442 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,442 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,442 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,443 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,443 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,443 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,449 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,449 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,450 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48dede73c5017e00f25348e75653b620, server=51ca66f7ee3c,36001,1731358281842 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,450 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T20:51:28,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,451 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,451 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,453 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,453 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,453 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,453 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,454 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,454 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318454 2024-11-11T20:51:28,454 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,454 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,455 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,456 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,456 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,457 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,462 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,463 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,464 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,464 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348464 2024-11-11T20:51:28,464 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408464 2024-11-11T20:51:28,465 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-11T20:51:28,465 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,465 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,465 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,465 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,465 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,465 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,465 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,466 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,469 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,470 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,471 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,474 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN 2024-11-11T20:51:28,474 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-11T20:51:28,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,476 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,476 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,478 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,478 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,478 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,478 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,478 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,478 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,479 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,479 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1c7ea4c5037945c7e8566334a0de9a50, ASSIGN to state=OFFLINE, location=null, table=test, region=1c7ea4c5037945c7e8566334a0de9a50 to restore RIT 2024-11-11T20:51:28,479 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318479 2024-11-11T20:51:28,479 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,479 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,481 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,482 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,482 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,482 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,489 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,489 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,489 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348489 2024-11-11T20:51:28,489 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408489 2024-11-11T20:51:28,489 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-11T20:51:28,489 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,489 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,489 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,489 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,489 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,489 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,490 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,490 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,491 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,493 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,494 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,494 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,496 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,496 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,496 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,497 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN 2024-11-11T20:51:28,497 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T20:51:28,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,498 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,498 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,500 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,500 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,500 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,500 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,500 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d414ef171f23d90cb3b6da2b30e93f36, ASSIGN to state=OFFLINE, location=null, table=test, region=d414ef171f23d90cb3b6da2b30e93f36 to restore RIT 2024-11-11T20:51:28,501 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318501 2024-11-11T20:51:28,501 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,501 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,503 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,506 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,506 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,506 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,506 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,508 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,511 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,511 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348511 2024-11-11T20:51:28,511 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408511 2024-11-11T20:51:28,511 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-11T20:51:28,511 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,511 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,511 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,511 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,512 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,512 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,512 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,513 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,516 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,517 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,517 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,520 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN 2024-11-11T20:51:28,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-11T20:51:28,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,523 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,523 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,525 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,525 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,525 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,525 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,525 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,526 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,526 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,526 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,526 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=07a697e4da0c9ded22ab8014e5ed7708, ASSIGN to state=OFFLINE, location=null, table=test, region=07a697e4da0c9ded22ab8014e5ed7708 to restore RIT 2024-11-11T20:51:28,526 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318526 2024-11-11T20:51:28,526 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,526 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,528 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,529 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,535 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348535 2024-11-11T20:51:28,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408535 2024-11-11T20:51:28,535 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-11T20:51:28,535 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,535 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,535 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,535 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,535 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,535 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,535 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,535 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,536 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,537 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,539 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,540 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,540 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,540 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,540 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,541 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,541 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,541 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,541 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,541 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,542 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN 2024-11-11T20:51:28,542 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T20:51:28,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,543 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,543 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,545 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=201d6d0f92a6319d987b626f2b7bad8b, ASSIGN to state=OFFLINE, location=null, table=test, region=201d6d0f92a6319d987b626f2b7bad8b to restore RIT 2024-11-11T20:51:28,546 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318546 2024-11-11T20:51:28,546 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,546 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,547 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,548 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,548 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,548 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,550 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,551 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,552 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,553 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,553 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,553 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348553 2024-11-11T20:51:28,553 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408553 2024-11-11T20:51:28,553 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-11T20:51:28,554 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,554 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,554 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,554 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,554 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,554 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,554 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,554 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,555 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,559 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,559 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,559 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,559 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN 2024-11-11T20:51:28,560 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T20:51:28,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,560 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,561 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,562 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,563 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,563 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,563 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,563 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=48dede73c5017e00f25348e75653b620, ASSIGN to state=OFFLINE, location=null, table=test, region=48dede73c5017e00f25348e75653b620 to restore RIT 2024-11-11T20:51:28,563 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318563 2024-11-11T20:51:28,563 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,563 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,565 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,566 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,566 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,566 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,569 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,570 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,571 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,571 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,571 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348571 2024-11-11T20:51:28,571 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408571 2024-11-11T20:51:28,571 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-11T20:51:28,571 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,572 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,572 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,572 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,572 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,572 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,572 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,572 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,575 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,576 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,578 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,578 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,579 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,579 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,579 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN 2024-11-11T20:51:28,579 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-11T20:51:28,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,580 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,580 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,582 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=084ef38b701ab7eeaa42a673a92ec1d6, ASSIGN to state=OFFLINE, location=null, table=test, region=084ef38b701ab7eeaa42a673a92ec1d6 to restore RIT 2024-11-11T20:51:28,590 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318590 2024-11-11T20:51:28,590 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,590 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,592 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,593 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,593 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,599 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,599 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348599 2024-11-11T20:51:28,599 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408599 2024-11-11T20:51:28,599 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-11T20:51:28,599 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,599 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,599 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,599 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,599 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,600 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,600 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,605 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,605 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,605 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN 2024-11-11T20:51:28,605 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T20:51:28,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,606 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,606 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,609 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,609 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f749f9c70b714192bc4197ca43f0d35e, ASSIGN to state=OFFLINE, location=null, table=test, region=f749f9c70b714192bc4197ca43f0d35e to restore RIT 2024-11-11T20:51:28,609 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318609 2024-11-11T20:51:28,609 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,609 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,611 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,612 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,618 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,618 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348618 2024-11-11T20:51:28,618 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408618 2024-11-11T20:51:28,618 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-11T20:51:28,619 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,619 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,619 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,619 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,619 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,619 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,619 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,619 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,620 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,623 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,624 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,625 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,625 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,625 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN 2024-11-11T20:51:28,625 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T20:51:28,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,626 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,626 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,628 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=885e30f0459dab7e5a3e27492bed3f2a, ASSIGN to state=OFFLINE, location=null, table=test, region=885e30f0459dab7e5a3e27492bed3f2a to restore RIT 2024-11-11T20:51:28,630 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318630 2024-11-11T20:51:28,630 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,630 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,632 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,635 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,635 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,635 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,637 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,637 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348637 2024-11-11T20:51:28,637 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408637 2024-11-11T20:51:28,638 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-11T20:51:28,638 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,638 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,638 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,638 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,638 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,639 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,639 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,639 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,639 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,640 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,640 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,642 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,643 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,644 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,644 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-11T20:51:28,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,644 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,645 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,647 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318647 2024-11-11T20:51:28,647 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,647 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,648 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,650 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,650 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,655 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,655 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,655 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348655 2024-11-11T20:51:28,655 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408655 2024-11-11T20:51:28,655 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-11T20:51:28,655 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,655 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,655 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,656 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,656 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,656 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,656 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,657 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,658 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,658 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,658 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,658 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,658 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,658 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,658 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.1731358283129, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-11T20:51:28,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,661 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,661 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,662 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,663 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,664 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318663 2024-11-11T20:51:28,664 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,664 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,665 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,666 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,666 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,667 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,672 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,672 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348672 2024-11-11T20:51:28,672 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408672 2024-11-11T20:51:28,672 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-11T20:51:28,672 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,672 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,672 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,673 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,673 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,673 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,673 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-11T20:51:28,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,676 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,677 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,677 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-11T20:51:28,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,678 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,678 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,680 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,680 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,680 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,681 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318681 2024-11-11T20:51:28,681 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,681 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,682 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,685 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,685 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,686 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,687 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,688 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,689 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,689 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348689 2024-11-11T20:51:28,689 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408689 2024-11-11T20:51:28,689 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-11T20:51:28,689 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,689 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,689 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,690 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,690 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,690 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,690 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,691 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-11T20:51:28,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:51:28,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T20:51:28,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,696 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,696 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,698 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,698 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,698 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,699 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318699 2024-11-11T20:51:28,699 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,699 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,701 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,701 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,702 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,707 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,707 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348707 2024-11-11T20:51:28,707 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408707 2024-11-11T20:51:28,707 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-11T20:51:28,707 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,707 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,707 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,707 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,707 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,707 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,707 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,708 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,708 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-11T20:51:28,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-11T20:51:28,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,712 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,712 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,714 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,715 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318715 2024-11-11T20:51:28,715 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,715 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,716 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,717 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,717 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,717 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,723 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348723 2024-11-11T20:51:28,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408723 2024-11-11T20:51:28,723 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-11T20:51:28,723 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,723 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,724 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,724 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,724 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,724 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,724 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,726 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,726 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-11T20:51:28,726 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,727 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,727 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,727 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 51ca66f7ee3c%2C38917%2C1731358281807.meta.1731358283631.meta, worker=51ca66f7ee3c,36213,1731358281724 didn't hold the lock before restarting, skip acquiring lock. 2024-11-11T20:51:28,727 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-11T20:51:28,728 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,728 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,728 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,730 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,731 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318731 2024-11-11T20:51:28,731 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,731 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,733 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,734 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-11T20:51:28,734 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,734 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,740 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,740 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348740 2024-11-11T20:51:28,740 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408740 2024-11-11T20:51:28,740 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-11T20:51:28,740 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-11T20:51:28,740 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,740 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:28,740 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:28,741 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-11T20:51:28,741 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-11T20:51:28,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-11T20:51:28,741 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:51:28,742 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T20:51:28,742 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,743 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38185 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,744 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-11T20:51:28,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:51:28,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-11T20:51:28,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true 2024-11-11T20:51:28,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-11T20:51:28,745 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-11T20:51:28,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=61 2024-11-11T20:51:28,746 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=51ca66f7ee3c,36213,1731358281724, table=hbase:meta, region=1588230740 2024-11-11T20:51:28,746 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,36213,1731358281724, state=OPEN 2024-11-11T20:51:28,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:51:28,748 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,748 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,748 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:51:28,750 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358318750 2024-11-11T20:51:28,750 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:51:28,750 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:51:28,752 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-11T20:51:28,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=75, ppid=66, state=RUNNABLE, hasLock=true; OpenRegionProcedure 084ef38b701ab7eeaa42a673a92ec1d6, server=51ca66f7ee3c,36001,1731358281842:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 51ca66f7ee3c,38917,1731358281807, splitWal=true, meta=true exec-time=2.2350 sec 2024-11-11T20:51:28,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f749f9c70b714192bc4197ca43f0d35e, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c574684bc1a84767063943424c84e3f2, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a71397de63e8cb71cb808d7888d245, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eef5c8934a3876cee983f4197f34ce96, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=caaf3dccceaaa47724f8b76041286a47, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=48dede73c5017e00f25348e75653b620, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fec0270188c1b9e90ffcce7addd0cf30, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9dfd755f05156eb757bfa3d4fdc2b62a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=885e30f0459dab7e5a3e27492bed3f2a, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4a56b479883cb03a610f18bcad1bfcc3, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=96c53e851b8fac62f296d140948ca3ee, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=084ef38b701ab7eeaa42a673a92ec1d6, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b414e49a28b2cae7a6a20bc170cff4f, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e6f7fe1f29c3816146fbdd9ef0338b39, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a298d35593aa3585cf7f54bd6d37af4, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bce7c26d0c06e82319102e4a44901ffa, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1c7ea4c5037945c7e8566334a0de9a50, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e0eaf6d4b3f10b135b25df3514191b, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43d3520c27ced34912ce528d771ce465, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07a697e4da0c9ded22ab8014e5ed7708, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5acc05cce6a62ba139f97712c529e120, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f43990b9e210c622161ccc0992d6c911, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=201d6d0f92a6319d987b626f2b7bad8b, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8b6777ce113fcb1df22e25ffb49623a, regionState=OPEN, lastHost=51ca66f7ee3c,36001,1731358281842, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd0c630a3962dac6e1875877d78d1d43, regionState=OPEN, lastHost=51ca66f7ee3c,36213,1731358281724, regionLocation=51ca66f7ee3c,36213,1731358281724, openSeqNum=2 2024-11-11T20:51:28,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d414ef171f23d90cb3b6da2b30e93f36, regionState=OPENING, lastHost=51ca66f7ee3c,38917,1731358281807, regionLocation=51ca66f7ee3c,36001,1731358281842, openSeqNum=2 2024-11-11T20:51:28,759 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T20:51:28,759 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358348759 2024-11-11T20:51:28,759 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358408759 2024-11-11T20:51:28,759 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-11T20:51:28,777 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=345 (was 380), OpenFileDescriptor=624 (was 647), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=103 (was 103), ProcessCount=11 (was 11), AvailableMemoryMB=4420 (was 4457) 2024-11-11T20:51:28,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T20:51:28,777 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:51:28,778 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:51:28,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:28,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:28,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:51:28,779 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T20:51:28,779 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1133720664, stopped=false 2024-11-11T20:51:28,779 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=51ca66f7ee3c,38185,1731358281139 2024-11-11T20:51:28,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:51:28,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:51:28,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:51:28,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:28,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:28,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:28,780 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:28,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:51:28,781 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:51:28,781 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:51:28,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:28,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:51:28,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:51:28,781 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,36213,1731358281724' ***** 2024-11-11T20:51:28,781 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T20:51:28,782 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,36001,1731358281842' ***** 2024-11-11T20:51:28,782 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T20:51:28,782 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:51:28,782 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:51:28,782 INFO [RS:2;51ca66f7ee3c:36001 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:51:28,782 INFO [RS:0;51ca66f7ee3c:36213 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:51:28,782 INFO [RS:2;51ca66f7ee3c:36001 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:51:28,782 INFO [RS:0;51ca66f7ee3c:36213 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:51:28,782 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:51:28,782 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:51:28,782 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for e6f7fe1f29c3816146fbdd9ef0338b39 2024-11-11T20:51:28,782 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for 96c53e851b8fac62f296d140948ca3ee 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 48dede73c5017e00f25348e75653b620 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for bce7c26d0c06e82319102e4a44901ffa 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 084ef38b701ab7eeaa42a673a92ec1d6 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for bd0c630a3962dac6e1875877d78d1d43 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 0a298d35593aa3585cf7f54bd6d37af4 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 9dfd755f05156eb757bfa3d4fdc2b62a 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for 885e30f0459dab7e5a3e27492bed3f2a 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for d8b6777ce113fcb1df22e25ffb49623a 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for caaf3dccceaaa47724f8b76041286a47 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 5acc05cce6a62ba139f97712c529e120 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for f43990b9e210c622161ccc0992d6c911 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 57a71397de63e8cb71cb808d7888d245 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for 43d3520c27ced34912ce528d771ce465 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for d414ef171f23d90cb3b6da2b30e93f36 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for c574684bc1a84767063943424c84e3f2 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for d0e0eaf6d4b3f10b135b25df3514191b 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 07a697e4da0c9ded22ab8014e5ed7708 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 0b414e49a28b2cae7a6a20bc170cff4f 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 4a56b479883cb03a610f18bcad1bfcc3 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:51:28,783 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e6f7fe1f29c3816146fbdd9ef0338b39, disabling compactions & flushes 2024-11-11T20:51:28,783 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eef5c8934a3876cee983f4197f34ce96, disabling compactions & flushes 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:28,783 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:28,783 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:28,783 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:51:28,783 INFO [RS:0;51ca66f7ee3c:36213 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;51ca66f7ee3c:36213. 2024-11-11T20:51:28,784 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:28,784 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:28,784 INFO [RS:2;51ca66f7ee3c:36001 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;51ca66f7ee3c:36001. 2024-11-11T20:51:28,784 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. after waiting 0 ms 2024-11-11T20:51:28,784 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. after waiting 0 ms 2024-11-11T20:51:28,784 DEBUG [RS:0;51ca66f7ee3c:36213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:51:28,784 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:28,784 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:28,784 DEBUG [RS:2;51ca66f7ee3c:36001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:51:28,784 DEBUG [RS:0;51ca66f7ee3c:36213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:28,784 DEBUG [RS:2;51ca66f7ee3c:36001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:28,784 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-11T20:51:28,784 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:51:28,784 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:51:28,784 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:51:28,784 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T20:51:28,785 DEBUG [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(1351): Waiting on 07a697e4da0c9ded22ab8014e5ed7708, 084ef38b701ab7eeaa42a673a92ec1d6, 0a298d35593aa3585cf7f54bd6d37af4, 0b414e49a28b2cae7a6a20bc170cff4f, 48dede73c5017e00f25348e75653b620, 4a56b479883cb03a610f18bcad1bfcc3, 57a71397de63e8cb71cb808d7888d245, 5acc05cce6a62ba139f97712c529e120, 9dfd755f05156eb757bfa3d4fdc2b62a, d0e0eaf6d4b3f10b135b25df3514191b, d414ef171f23d90cb3b6da2b30e93f36, d8b6777ce113fcb1df22e25ffb49623a, eef5c8934a3876cee983f4197f34ce96 2024-11-11T20:51:28,785 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-11T20:51:28,785 DEBUG [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1c7ea4c5037945c7e8566334a0de9a50, 43d3520c27ced34912ce528d771ce465, 885e30f0459dab7e5a3e27492bed3f2a, 96c53e851b8fac62f296d140948ca3ee, bce7c26d0c06e82319102e4a44901ffa, bd0c630a3962dac6e1875877d78d1d43, c574684bc1a84767063943424c84e3f2, caaf3dccceaaa47724f8b76041286a47, e6f7fe1f29c3816146fbdd9ef0338b39, f43990b9e210c622161ccc0992d6c911, fec0270188c1b9e90ffcce7addd0cf30 2024-11-11T20:51:28,785 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:51:28,785 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:51:28,785 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:51:28,785 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:51:28,785 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:51:28,786 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-11T20:51:28,790 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/eef5c8934a3876cee983f4197f34ce96/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,792 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:28,792 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eef5c8934a3876cee983f4197f34ce96: Waiting for close lock at 1731358288783Running coprocessor pre-close hooks at 1731358288783Disabling compacts and flushes for region at 1731358288783Disabling writes for close at 1731358288784 (+1 ms)Writing region close event to WAL at 1731358288785 (+1 ms)Running coprocessor post-close hooks at 1731358288790 (+5 ms)Closed at 1731358288792 (+2 ms) 2024-11-11T20:51:28,792 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/e6f7fe1f29c3816146fbdd9ef0338b39/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,793 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731358284091.eef5c8934a3876cee983f4197f34ce96. 2024-11-11T20:51:28,793 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 48dede73c5017e00f25348e75653b620, disabling compactions & flushes 2024-11-11T20:51:28,793 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:28,793 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:28,793 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731358284091.48dede73c5017e00f25348e75653b620. after waiting 0 ms 2024-11-11T20:51:28,793 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:28,793 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:28,793 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e6f7fe1f29c3816146fbdd9ef0338b39: Waiting for close lock at 1731358288783Running coprocessor pre-close hooks at 1731358288783Disabling compacts and flushes for region at 1731358288783Disabling writes for close at 1731358288784 (+1 ms)Writing region close event to WAL at 1731358288785 (+1 ms)Running coprocessor post-close hooks at 1731358288793 (+8 ms)Closed at 1731358288793 2024-11-11T20:51:28,793 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731358284091.e6f7fe1f29c3816146fbdd9ef0338b39. 2024-11-11T20:51:28,794 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 96c53e851b8fac62f296d140948ca3ee, disabling compactions & flushes 2024-11-11T20:51:28,794 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:28,794 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:28,794 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. after waiting 0 ms 2024-11-11T20:51:28,794 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:28,799 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/48dede73c5017e00f25348e75653b620/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,799 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/96c53e851b8fac62f296d140948ca3ee/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,800 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 48dede73c5017e00f25348e75653b620: Waiting for close lock at 1731358288793Running coprocessor pre-close hooks at 1731358288793Disabling compacts and flushes for region at 1731358288793Disabling writes for close at 1731358288793Writing region close event to WAL at 1731358288794 (+1 ms)Running coprocessor post-close hooks at 1731358288799 (+5 ms)Closed at 1731358288800 (+1 ms) 2024-11-11T20:51:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731358284091.48dede73c5017e00f25348e75653b620. 2024-11-11T20:51:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 084ef38b701ab7eeaa42a673a92ec1d6, disabling compactions & flushes 2024-11-11T20:51:28,800 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. after waiting 0 ms 2024-11-11T20:51:28,800 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:28,801 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:28,801 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 96c53e851b8fac62f296d140948ca3ee: Waiting for close lock at 1731358288794Running coprocessor pre-close hooks at 1731358288794Disabling compacts and flushes for region at 1731358288794Disabling writes for close at 1731358288794Writing region close event to WAL at 1731358288795 (+1 ms)Running coprocessor post-close hooks at 1731358288801 (+6 ms)Closed at 1731358288801 2024-11-11T20:51:28,801 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731358284091.96c53e851b8fac62f296d140948ca3ee. 2024-11-11T20:51:28,801 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/info/4358eca97b5d4427bbb9e31ef119fc56 is 138, key is test,fff,1731358284091.48dede73c5017e00f25348e75653b620./info:regioninfo/1731358287893/Put/seqid=0 2024-11-11T20:51:28,801 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bce7c26d0c06e82319102e4a44901ffa, disabling compactions & flushes 2024-11-11T20:51:28,801 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:28,801 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:28,801 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. after waiting 0 ms 2024-11-11T20:51:28,801 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:28,807 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/084ef38b701ab7eeaa42a673a92ec1d6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,807 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/bce7c26d0c06e82319102e4a44901ffa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,808 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:28,808 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 084ef38b701ab7eeaa42a673a92ec1d6: Waiting for close lock at 1731358288800Running coprocessor pre-close hooks at 1731358288800Disabling compacts and flushes for region at 1731358288800Disabling writes for close at 1731358288800Writing region close event to WAL at 1731358288801 (+1 ms)Running coprocessor post-close hooks at 1731358288808 (+7 ms)Closed at 1731358288808 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bce7c26d0c06e82319102e4a44901ffa: Waiting for close lock at 1731358288801Running coprocessor pre-close hooks at 1731358288801Disabling compacts and flushes for region at 1731358288801Disabling writes for close at 1731358288801Writing region close event to WAL at 1731358288802 (+1 ms)Running coprocessor post-close hooks at 1731358288808 (+6 ms)Closed at 1731358288808 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731358284091.084ef38b701ab7eeaa42a673a92ec1d6. 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731358284091.bce7c26d0c06e82319102e4a44901ffa. 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd0c630a3962dac6e1875877d78d1d43, disabling compactions & flushes 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0a298d35593aa3585cf7f54bd6d37af4, disabling compactions & flushes 2024-11-11T20:51:28,808 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:28,808 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. after waiting 0 ms 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. after waiting 0 ms 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:28,808 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:28,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741869_1045 (size=8174) 2024-11-11T20:51:28,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741869_1045 (size=8174) 2024-11-11T20:51:28,817 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/info/4358eca97b5d4427bbb9e31ef119fc56 2024-11-11T20:51:28,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741869_1045 (size=8174) 2024-11-11T20:51:28,817 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/0a298d35593aa3585cf7f54bd6d37af4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,818 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/bd0c630a3962dac6e1875877d78d1d43/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,818 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:28,818 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd0c630a3962dac6e1875877d78d1d43: Waiting for close lock at 1731358288808Running coprocessor pre-close hooks at 1731358288808Disabling compacts and flushes for region at 1731358288808Disabling writes for close at 1731358288808Writing region close event to WAL at 1731358288812 (+4 ms)Running coprocessor post-close hooks at 1731358288818 (+6 ms)Closed at 1731358288818 2024-11-11T20:51:28,819 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731358284091.bd0c630a3962dac6e1875877d78d1d43. 2024-11-11T20:51:28,819 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fec0270188c1b9e90ffcce7addd0cf30, disabling compactions & flushes 2024-11-11T20:51:28,819 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:28,819 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:28,819 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. after waiting 0 ms 2024-11-11T20:51:28,819 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:28,819 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:28,819 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0a298d35593aa3585cf7f54bd6d37af4: Waiting for close lock at 1731358288808Running coprocessor pre-close hooks at 1731358288808Disabling compacts and flushes for region at 1731358288808Disabling writes for close at 1731358288808Writing region close event to WAL at 1731358288812 (+4 ms)Running coprocessor post-close hooks at 1731358288819 (+7 ms)Closed at 1731358288819 2024-11-11T20:51:28,819 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731358284091.0a298d35593aa3585cf7f54bd6d37af4. 2024-11-11T20:51:28,820 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9dfd755f05156eb757bfa3d4fdc2b62a, disabling compactions & flushes 2024-11-11T20:51:28,820 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:28,820 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:28,820 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. after waiting 0 ms 2024-11-11T20:51:28,820 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:28,826 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/9dfd755f05156eb757bfa3d4fdc2b62a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,826 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/fec0270188c1b9e90ffcce7addd0cf30/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,827 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:28,827 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9dfd755f05156eb757bfa3d4fdc2b62a: Waiting for close lock at 1731358288820Running coprocessor pre-close hooks at 1731358288820Disabling compacts and flushes for region at 1731358288820Disabling writes for close at 1731358288820Writing region close event to WAL at 1731358288820Running coprocessor post-close hooks at 1731358288826 (+6 ms)Closed at 1731358288827 (+1 ms) 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fec0270188c1b9e90ffcce7addd0cf30: Waiting for close lock at 1731358288819Running coprocessor pre-close hooks at 1731358288819Disabling compacts and flushes for region at 1731358288819Disabling writes for close at 1731358288819Writing region close event to WAL at 1731358288820 (+1 ms)Running coprocessor post-close hooks at 1731358288827 (+7 ms)Closed at 1731358288827 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731358284091.9dfd755f05156eb757bfa3d4fdc2b62a. 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731358284091.fec0270188c1b9e90ffcce7addd0cf30. 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d8b6777ce113fcb1df22e25ffb49623a, disabling compactions & flushes 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 885e30f0459dab7e5a3e27492bed3f2a, disabling compactions & flushes 2024-11-11T20:51:28,827 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:28,827 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. after waiting 0 ms 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. after waiting 0 ms 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:28,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:28,828 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/.tmp/info/4358eca97b5d4427bbb9e31ef119fc56 as hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/info/4358eca97b5d4427bbb9e31ef119fc56 2024-11-11T20:51:28,832 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/885e30f0459dab7e5a3e27492bed3f2a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,833 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d8b6777ce113fcb1df22e25ffb49623a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,833 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 885e30f0459dab7e5a3e27492bed3f2a: Waiting for close lock at 1731358288827Running coprocessor pre-close hooks at 1731358288827Disabling compacts and flushes for region at 1731358288827Disabling writes for close at 1731358288827Writing region close event to WAL at 1731358288828 (+1 ms)Running coprocessor post-close hooks at 1731358288833 (+5 ms)Closed at 1731358288833 2024-11-11T20:51:28,834 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d8b6777ce113fcb1df22e25ffb49623a: Waiting for close lock at 1731358288827Running coprocessor pre-close hooks at 1731358288827Disabling compacts and flushes for region at 1731358288827Disabling writes for close at 1731358288827Writing region close event to WAL at 1731358288827Running coprocessor post-close hooks at 1731358288834 (+7 ms)Closed at 1731358288834 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731358284091.885e30f0459dab7e5a3e27492bed3f2a. 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731358284091.d8b6777ce113fcb1df22e25ffb49623a. 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5acc05cce6a62ba139f97712c529e120, disabling compactions & flushes 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing caaf3dccceaaa47724f8b76041286a47, disabling compactions & flushes 2024-11-11T20:51:28,834 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:28,834 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. after waiting 0 ms 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. after waiting 0 ms 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:28,834 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:28,839 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/info/4358eca97b5d4427bbb9e31ef119fc56, entries=24, sequenceid=71, filesize=8.0 K 2024-11-11T20:51:28,841 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2705, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 55ms, sequenceid=71, compaction requested=false 2024-11-11T20:51:28,841 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/5acc05cce6a62ba139f97712c529e120/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/caaf3dccceaaa47724f8b76041286a47/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,842 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5acc05cce6a62ba139f97712c529e120: Waiting for close lock at 1731358288834Running coprocessor pre-close hooks at 1731358288834Disabling compacts and flushes for region at 1731358288834Disabling writes for close at 1731358288834Writing region close event to WAL at 1731358288835 (+1 ms)Running coprocessor post-close hooks at 1731358288842 (+7 ms)Closed at 1731358288842 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731358284091.5acc05cce6a62ba139f97712c529e120. 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 57a71397de63e8cb71cb808d7888d245, disabling compactions & flushes 2024-11-11T20:51:28,842 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:28,842 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. after waiting 0 ms 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for caaf3dccceaaa47724f8b76041286a47: Waiting for close lock at 1731358288834Running coprocessor pre-close hooks at 1731358288834Disabling compacts and flushes for region at 1731358288834Disabling writes for close at 1731358288834Writing region close event to WAL at 1731358288835 (+1 ms)Running coprocessor post-close hooks at 1731358288842 (+7 ms)Closed at 1731358288842 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731358284091.caaf3dccceaaa47724f8b76041286a47. 2024-11-11T20:51:28,842 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f43990b9e210c622161ccc0992d6c911, disabling compactions & flushes 2024-11-11T20:51:28,843 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:28,843 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:28,843 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. after waiting 0 ms 2024-11-11T20:51:28,843 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:28,851 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/57a71397de63e8cb71cb808d7888d245/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,851 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/f43990b9e210c622161ccc0992d6c911/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,852 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:28,852 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f43990b9e210c622161ccc0992d6c911: Waiting for close lock at 1731358288842Running coprocessor pre-close hooks at 1731358288842Disabling compacts and flushes for region at 1731358288842Disabling writes for close at 1731358288843 (+1 ms)Writing region close event to WAL at 1731358288847 (+4 ms)Running coprocessor post-close hooks at 1731358288851 (+4 ms)Closed at 1731358288852 (+1 ms) 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 57a71397de63e8cb71cb808d7888d245: Waiting for close lock at 1731358288842Running coprocessor pre-close hooks at 1731358288842Disabling compacts and flushes for region at 1731358288842Disabling writes for close at 1731358288842Writing region close event to WAL at 1731358288847 (+5 ms)Running coprocessor post-close hooks at 1731358288852 (+5 ms)Closed at 1731358288852 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731358284091.57a71397de63e8cb71cb808d7888d245. 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731358284091.f43990b9e210c622161ccc0992d6c911. 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d414ef171f23d90cb3b6da2b30e93f36, disabling compactions & flushes 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 43d3520c27ced34912ce528d771ce465, disabling compactions & flushes 2024-11-11T20:51:28,852 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:28,852 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. after waiting 0 ms 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. after waiting 0 ms 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:28,852 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-11T20:51:28,853 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:51:28,854 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:51:28,854 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358288785Running coprocessor pre-close hooks at 1731358288785Disabling compacts and flushes for region at 1731358288785Disabling writes for close at 1731358288785Obtaining lock to block concurrent updates at 1731358288786 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731358288786Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2705, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1731358288794 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731358288795 (+1 ms)Flushing 1588230740/info: creating writer at 1731358288795Flushing 1588230740/info: appending metadata at 1731358288801 (+6 ms)Flushing 1588230740/info: closing flushed file at 1731358288801Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@490aa2a3: reopening flushed file at 1731358288827 (+26 ms)Finished flush of dataSize ~2.64 KB/2705, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 55ms, sequenceid=71, compaction requested=false at 1731358288841 (+14 ms)Writing region close event to WAL at 1731358288848 (+7 ms)Running coprocessor post-close hooks at 1731358288853 (+5 ms)Closed at 1731358288854 (+1 ms) 2024-11-11T20:51:28,854 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T20:51:28,856 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d414ef171f23d90cb3b6da2b30e93f36/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,856 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/43d3520c27ced34912ce528d771ce465/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,857 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d414ef171f23d90cb3b6da2b30e93f36: Waiting for close lock at 1731358288852Running coprocessor pre-close hooks at 1731358288852Disabling compacts and flushes for region at 1731358288852Disabling writes for close at 1731358288852Writing region close event to WAL at 1731358288853 (+1 ms)Running coprocessor post-close hooks at 1731358288857 (+4 ms)Closed at 1731358288857 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731358284091.d414ef171f23d90cb3b6da2b30e93f36. 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d0e0eaf6d4b3f10b135b25df3514191b, disabling compactions & flushes 2024-11-11T20:51:28,857 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:28,857 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 43d3520c27ced34912ce528d771ce465: Waiting for close lock at 1731358288852Running coprocessor pre-close hooks at 1731358288852Disabling compacts and flushes for region at 1731358288852Disabling writes for close at 1731358288852Writing region close event to WAL at 1731358288853 (+1 ms)Running coprocessor post-close hooks at 1731358288857 (+4 ms)Closed at 1731358288857 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731358284091.43d3520c27ced34912ce528d771ce465. 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. after waiting 0 ms 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c574684bc1a84767063943424c84e3f2, disabling compactions & flushes 2024-11-11T20:51:28,857 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:28,857 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. after waiting 0 ms 2024-11-11T20:51:28,858 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:28,861 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/c574684bc1a84767063943424c84e3f2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,861 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/d0e0eaf6d4b3f10b135b25df3514191b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,862 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:51:28,862 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:28,862 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:28,862 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d0e0eaf6d4b3f10b135b25df3514191b: Waiting for close lock at 1731358288857Running coprocessor pre-close hooks at 1731358288857Disabling compacts and flushes for region at 1731358288857Disabling writes for close at 1731358288857Writing region close event to WAL at 1731358288858 (+1 ms)Running coprocessor post-close hooks at 1731358288862 (+4 ms)Closed at 1731358288862 2024-11-11T20:51:28,862 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c574684bc1a84767063943424c84e3f2: Waiting for close lock at 1731358288857Running coprocessor pre-close hooks at 1731358288857Disabling compacts and flushes for region at 1731358288857Disabling writes for close at 1731358288858 (+1 ms)Writing region close event to WAL at 1731358288858Running coprocessor post-close hooks at 1731358288862 (+4 ms)Closed at 1731358288862 2024-11-11T20:51:28,862 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731358284091.c574684bc1a84767063943424c84e3f2. 2024-11-11T20:51:28,862 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731358284091.d0e0eaf6d4b3f10b135b25df3514191b. 2024-11-11T20:51:28,862 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 07a697e4da0c9ded22ab8014e5ed7708, disabling compactions & flushes 2024-11-11T20:51:28,862 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:28,862 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:28,862 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. after waiting 0 ms 2024-11-11T20:51:28,862 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:28,867 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/07a697e4da0c9ded22ab8014e5ed7708/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,867 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:28,867 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 07a697e4da0c9ded22ab8014e5ed7708: Waiting for close lock at 1731358288862Running coprocessor pre-close hooks at 1731358288862Disabling compacts and flushes for region at 1731358288862Disabling writes for close at 1731358288862Writing region close event to WAL at 1731358288863 (+1 ms)Running coprocessor post-close hooks at 1731358288867 (+4 ms)Closed at 1731358288867 2024-11-11T20:51:28,867 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731358284091.07a697e4da0c9ded22ab8014e5ed7708. 2024-11-11T20:51:28,868 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0b414e49a28b2cae7a6a20bc170cff4f, disabling compactions & flushes 2024-11-11T20:51:28,868 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:28,868 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:28,868 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. after waiting 0 ms 2024-11-11T20:51:28,868 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:28,868 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:51:28,872 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/0b414e49a28b2cae7a6a20bc170cff4f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,872 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:28,872 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0b414e49a28b2cae7a6a20bc170cff4f: Waiting for close lock at 1731358288867Running coprocessor pre-close hooks at 1731358288867Disabling compacts and flushes for region at 1731358288867Disabling writes for close at 1731358288868 (+1 ms)Writing region close event to WAL at 1731358288868Running coprocessor post-close hooks at 1731358288872 (+4 ms)Closed at 1731358288872 2024-11-11T20:51:28,873 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731358284091.0b414e49a28b2cae7a6a20bc170cff4f. 2024-11-11T20:51:28,873 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4a56b479883cb03a610f18bcad1bfcc3, disabling compactions & flushes 2024-11-11T20:51:28,873 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:28,873 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:28,873 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. after waiting 0 ms 2024-11-11T20:51:28,873 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:28,877 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/data/default/test/4a56b479883cb03a610f18bcad1bfcc3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:51:28,878 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:28,878 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4a56b479883cb03a610f18bcad1bfcc3: Waiting for close lock at 1731358288873Running coprocessor pre-close hooks at 1731358288873Disabling compacts and flushes for region at 1731358288873Disabling writes for close at 1731358288873Writing region close event to WAL at 1731358288873Running coprocessor post-close hooks at 1731358288878 (+5 ms)Closed at 1731358288878 2024-11-11T20:51:28,878 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731358284091.4a56b479883cb03a610f18bcad1bfcc3. 2024-11-11T20:51:28,951 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T20:51:28,951 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T20:51:28,960 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T20:51:28,960 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T20:51:28,985 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,36001,1731358281842; all regions closed. 2024-11-11T20:51:28,985 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,36213,1731358281724; all regions closed. 2024-11-11T20:51:28,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741865_1041 (size=4189) 2024-11-11T20:51:28,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741865_1041 (size=4189) 2024-11-11T20:51:28,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741833_1009 (size=7074) 2024-11-11T20:51:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741833_1009 (size=7074) 2024-11-11T20:51:28,998 DEBUG [RS:0;51ca66f7ee3c:36213 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs 2024-11-11T20:51:28,998 DEBUG [RS:2;51ca66f7ee3c:36001 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs 2024-11-11T20:51:28,999 INFO [RS:0;51ca66f7ee3c:36213 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C36213%2C1731358281724.meta:.meta(num 1731358287143) 2024-11-11T20:51:28,999 INFO [RS:2;51ca66f7ee3c:36001 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C36001%2C1731358281842:(num 1731358283129) 2024-11-11T20:51:28,999 DEBUG [RS:2;51ca66f7ee3c:36001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:28,999 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:51:28,999 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:51:28,999 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T20:51:28,999 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:51:28,999 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:51:28,999 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:51:28,999 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:51:29,000 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:51:29,000 INFO [RS:2;51ca66f7ee3c:36001 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36001 2024-11-11T20:51:29,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741865_1041 (size=4189) 2024-11-11T20:51:29,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741833_1009 (size=7074) 2024-11-11T20:51:29,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:51:29,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,36001,1731358281842 2024-11-11T20:51:29,003 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:51:29,003 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,36001,1731358281842] 2024-11-11T20:51:29,004 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/WALs/51ca66f7ee3c,36213,1731358281724/51ca66f7ee3c%2C36213%2C1731358281724.1731358283129 not finished, retry = 0 2024-11-11T20:51:29,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741835_1011 (size=5731) 2024-11-11T20:51:29,004 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,36001,1731358281842 already deleted, retry=false 2024-11-11T20:51:29,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741835_1011 (size=5731) 2024-11-11T20:51:29,004 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51ca66f7ee3c,36001,1731358281842 expired; onlineServers=1 2024-11-11T20:51:29,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741835_1011 (size=5731) 2024-11-11T20:51:29,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:51:29,104 INFO [RS:2;51ca66f7ee3c:36001 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:51:29,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100308e27080003, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:51:29,104 INFO [RS:2;51ca66f7ee3c:36001 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,36001,1731358281842; zookeeper connection closed. 2024-11-11T20:51:29,105 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1d859fbd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1d859fbd 2024-11-11T20:51:29,111 DEBUG [RS:0;51ca66f7ee3c:36213 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/oldWALs 2024-11-11T20:51:29,111 INFO [RS:0;51ca66f7ee3c:36213 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C36213%2C1731358281724:(num 1731358283129) 2024-11-11T20:51:29,111 DEBUG [RS:0;51ca66f7ee3c:36213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:51:29,111 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:51:29,111 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:51:29,112 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T20:51:29,112 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:51:29,112 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:51:29,113 WARN [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:51:29,113 WARN [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50., aborting... java.io.IOException: Failed to report opened region to master: test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:51:29,114 ERROR [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server 51ca66f7ee3c,36213,1731358281724: Failed to open region test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,qqq,1731358284091.1c7ea4c5037945c7e8566334a0de9a50. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:51:29,114 ERROR [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-11T20:51:29,114 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-11T20:51:29,115 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-11T20:51:29,115 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-11T20:51:29,116 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-11T20:51:29,116 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 472717040 }, "NonHeapMemoryUsage": { "committed": 162463744, "init": 7667712, "max": -1, "used": 159862976 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-11-11T20:51:29,116 INFO [RS:0;51ca66f7ee3c:36213 {}] executor.ExecutorService(108): Executor-10-RS_OPEN_REGION-regionserver/51ca66f7ee3c:0 had [Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-11T20:51:29,116 INFO [RS:0;51ca66f7ee3c:36213 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36213 2024-11-11T20:51:29,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,36213,1731358281724 2024-11-11T20:51:29,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:51:29,118 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:51:29,119 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,36213,1731358281724] 2024-11-11T20:51:29,119 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,36213,1731358281724 already deleted, retry=false 2024-11-11T20:51:29,119 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51ca66f7ee3c,36213,1731358281724 expired; onlineServers=0 2024-11-11T20:51:29,120 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '51ca66f7ee3c,38185,1731358281139' ***** 2024-11-11T20:51:29,120 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T20:51:29,120 INFO [M:0;51ca66f7ee3c:38185 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:51:29,120 INFO [M:0;51ca66f7ee3c:38185 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:51:29,120 DEBUG [M:0;51ca66f7ee3c:38185 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T20:51:29,120 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T20:51:29,120 DEBUG [M:0;51ca66f7ee3c:38185 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T20:51:29,120 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358282755 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358282755,5,FailOnTimeoutGroup] 2024-11-11T20:51:29,120 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358282756 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358282756,5,FailOnTimeoutGroup] 2024-11-11T20:51:29,121 INFO [M:0;51ca66f7ee3c:38185 {}] hbase.ChoreService(370): Chore service for: master/51ca66f7ee3c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T20:51:29,121 INFO [M:0;51ca66f7ee3c:38185 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:51:29,121 DEBUG [M:0;51ca66f7ee3c:38185 {}] master.HMaster(1795): Stopping service threads 2024-11-11T20:51:29,121 INFO [M:0;51ca66f7ee3c:38185 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T20:51:29,121 INFO [M:0;51ca66f7ee3c:38185 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:51:29,121 INFO [M:0;51ca66f7ee3c:38185 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:51:29,121 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T20:51:29,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T20:51:29,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:51:29,122 DEBUG [M:0;51ca66f7ee3c:38185 {}] zookeeper.ZKUtil(347): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T20:51:29,122 WARN [M:0;51ca66f7ee3c:38185 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T20:51:29,122 INFO [M:0;51ca66f7ee3c:38185 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/.lastflushedseqids 2024-11-11T20:51:29,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741870_1046 (size=1) 2024-11-11T20:51:29,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741870_1046 (size=1) 2024-11-11T20:51:29,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741870_1046 (size=1) 2024-11-11T20:51:29,137 INFO [M:0;51ca66f7ee3c:38185 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:51:29,137 INFO [M:0;51ca66f7ee3c:38185 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T20:51:29,138 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:51:29,138 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:51:29,138 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:51:29,138 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:51:29,138 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:51:29,138 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=249.02 KB heapSize=302.61 KB 2024-11-11T20:51:29,156 DEBUG [M:0;51ca66f7ee3c:38185 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6a932c0256514bc68e69dcce0c523f1a is 82, key is hbase:meta,,1/info:regioninfo/1731358287489/Put/seqid=0 2024-11-11T20:51:29,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741871_1047 (size=6071) 2024-11-11T20:51:29,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741871_1047 (size=6071) 2024-11-11T20:51:29,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741871_1047 (size=6071) 2024-11-11T20:51:29,163 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6a932c0256514bc68e69dcce0c523f1a 2024-11-11T20:51:29,170 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:51:29,188 DEBUG [M:0;51ca66f7ee3c:38185 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/934d641d96bb4a149b1e46742c5af2a4 is 2984, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731358288752/Put/seqid=0 2024-11-11T20:51:29,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741872_1048 (size=12795) 2024-11-11T20:51:29,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741872_1048 (size=12795) 2024-11-11T20:51:29,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741872_1048 (size=12795) 2024-11-11T20:51:29,198 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=247.74 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/934d641d96bb4a149b1e46742c5af2a4 2024-11-11T20:51:29,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:51:29,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36213-0x100308e27080001, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:51:29,219 INFO [RS:0;51ca66f7ee3c:36213 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:51:29,219 INFO [RS:0;51ca66f7ee3c:36213 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,36213,1731358281724; zookeeper connection closed. 2024-11-11T20:51:29,219 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@647427b5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@647427b5 2024-11-11T20:51:29,220 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-11T20:51:29,222 DEBUG [M:0;51ca66f7ee3c:38185 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/878b1ad0f92944f8a126c454b36b49b7 is 69, key is 51ca66f7ee3c,36001,1731358281842/rs:state/1731358282846/Put/seqid=0 2024-11-11T20:51:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741873_1049 (size=5442) 2024-11-11T20:51:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741873_1049 (size=5442) 2024-11-11T20:51:29,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741873_1049 (size=5442) 2024-11-11T20:51:29,236 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-11T20:51:29,237 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/878b1ad0f92944f8a126c454b36b49b7 2024-11-11T20:51:29,244 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 878b1ad0f92944f8a126c454b36b49b7 2024-11-11T20:51:29,262 DEBUG [M:0;51ca66f7ee3c:38185 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ceb1ffb69bfb404bb95a98ad8d98808f is 52, key is load_balancer_on/state:d/1731358286434/Put/seqid=0 2024-11-11T20:51:29,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741874_1050 (size=5058) 2024-11-11T20:51:29,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741874_1050 (size=5058) 2024-11-11T20:51:29,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741874_1050 (size=5058) 2024-11-11T20:51:29,271 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ceb1ffb69bfb404bb95a98ad8d98808f 2024-11-11T20:51:29,280 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6a932c0256514bc68e69dcce0c523f1a as hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6a932c0256514bc68e69dcce0c523f1a 2024-11-11T20:51:29,288 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6a932c0256514bc68e69dcce0c523f1a, entries=14, sequenceid=572, filesize=5.9 K 2024-11-11T20:51:29,290 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/934d641d96bb4a149b1e46742c5af2a4 as hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/934d641d96bb4a149b1e46742c5af2a4 2024-11-11T20:51:29,297 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/934d641d96bb4a149b1e46742c5af2a4, entries=79, sequenceid=572, filesize=12.5 K 2024-11-11T20:51:29,299 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/878b1ad0f92944f8a126c454b36b49b7 as hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/878b1ad0f92944f8a126c454b36b49b7 2024-11-11T20:51:29,306 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 878b1ad0f92944f8a126c454b36b49b7 2024-11-11T20:51:29,306 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/878b1ad0f92944f8a126c454b36b49b7, entries=3, sequenceid=572, filesize=5.3 K 2024-11-11T20:51:29,307 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ceb1ffb69bfb404bb95a98ad8d98808f as hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ceb1ffb69bfb404bb95a98ad8d98808f 2024-11-11T20:51:29,315 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45383/user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ceb1ffb69bfb404bb95a98ad8d98808f, entries=1, sequenceid=572, filesize=4.9 K 2024-11-11T20:51:29,316 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(3140): Finished flush of dataSize ~249.02 KB/254995, heapSize ~302.55 KB/309808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 178ms, sequenceid=572, compaction requested=false 2024-11-11T20:51:29,317 INFO [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:51:29,317 DEBUG [M:0;51ca66f7ee3c:38185 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731358289138Disabling compacts and flushes for region at 1731358289138Disabling writes for close at 1731358289138Obtaining lock to block concurrent updates at 1731358289138Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731358289138Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254995, getHeapSize=309808, getOffHeapSize=0, getCellsCount=711 at 1731358289139 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731358289139Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731358289140 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731358289155 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731358289156 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731358289169 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731358289187 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731358289187Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731358289204 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731358289221 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731358289221Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731358289245 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731358289262 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731358289262Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16bafe8b: reopening flushed file at 1731358289279 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5983e1db: reopening flushed file at 1731358289288 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5143448c: reopening flushed file at 1731358289298 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c61f707: reopening flushed file at 1731358289306 (+8 ms)Finished flush of dataSize ~249.02 KB/254995, heapSize ~302.55 KB/309808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 178ms, sequenceid=572, compaction requested=false at 1731358289316 (+10 ms)Writing region close event to WAL at 1731358289317 (+1 ms)Closed at 1731358289317 2024-11-11T20:51:29,320 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7476ae64-349b-728b-2b4a-5b7671913e4a/MasterData/WALs/51ca66f7ee3c,38185,1731358281139/51ca66f7ee3c%2C38185%2C1731358281139.1731358282234 not finished, retry = 0 2024-11-11T20:51:29,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36815 is added to blk_1073741830_1006 (size=293184) 2024-11-11T20:51:29,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38355 is added to blk_1073741830_1006 (size=293184) 2024-11-11T20:51:29,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45587 is added to blk_1073741830_1006 (size=293184) 2024-11-11T20:51:29,422 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:51:29,422 INFO [M:0;51ca66f7ee3c:38185 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T20:51:29,422 INFO [M:0;51ca66f7ee3c:38185 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38185 2024-11-11T20:51:29,422 INFO [M:0;51ca66f7ee3c:38185 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:51:29,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:51:29,524 INFO [M:0;51ca66f7ee3c:38185 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:51:29,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38185-0x100308e27080000, quorum=127.0.0.1:57196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:51:29,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:51:29,534 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T20:51:29,534 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T20:51:29,534 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T20:51:29,534 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir/,STOPPED} 2024-11-11T20:51:29,537 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T20:51:29,537 WARN [BP-896987307-172.17.0.2-1731358278425 heartbeating to localhost/127.0.0.1:45383 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T20:51:29,537 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T20:51:29,537 WARN [BP-896987307-172.17.0.2-1731358278425 heartbeating to localhost/127.0.0.1:45383 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-896987307-172.17.0.2-1731358278425 (Datanode Uuid a166485b-082b-47f0-9dd4-85e666463258) service to localhost/127.0.0.1:45383 2024-11-11T20:51:29,538 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data5/current/BP-896987307-172.17.0.2-1731358278425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:51:29,539 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data6/current/BP-896987307-172.17.0.2-1731358278425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:51:29,539 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T20:51:29,550 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:51:29,550 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T20:51:29,551 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T20:51:29,551 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T20:51:29,551 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir/,STOPPED} 2024-11-11T20:51:29,552 WARN [BP-896987307-172.17.0.2-1731358278425 heartbeating to localhost/127.0.0.1:45383 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T20:51:29,552 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T20:51:29,552 WARN [BP-896987307-172.17.0.2-1731358278425 heartbeating to localhost/127.0.0.1:45383 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-896987307-172.17.0.2-1731358278425 (Datanode Uuid dae37a18-934f-4592-8acc-44b7bbdbc324) service to localhost/127.0.0.1:45383 2024-11-11T20:51:29,552 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T20:51:29,553 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data3/current/BP-896987307-172.17.0.2-1731358278425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:51:29,553 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data4/current/BP-896987307-172.17.0.2-1731358278425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:51:29,554 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T20:51:29,563 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:51:29,564 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T20:51:29,564 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T20:51:29,564 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T20:51:29,564 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir/,STOPPED} 2024-11-11T20:51:29,565 WARN [BP-896987307-172.17.0.2-1731358278425 heartbeating to localhost/127.0.0.1:45383 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T20:51:29,565 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T20:51:29,565 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T20:51:29,565 WARN [BP-896987307-172.17.0.2-1731358278425 heartbeating to localhost/127.0.0.1:45383 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-896987307-172.17.0.2-1731358278425 (Datanode Uuid 59034bab-165e-448c-8d98-6d0f088c67f5) service to localhost/127.0.0.1:45383 2024-11-11T20:51:29,566 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data1/current/BP-896987307-172.17.0.2-1731358278425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:51:29,567 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/cluster_1b502540-394e-6e4d-ea9c-cad70ec48a87/data/data2/current/BP-896987307-172.17.0.2-1731358278425 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:51:29,567 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T20:51:29,579 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T20:51:29,580 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T20:51:29,580 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T20:51:29,580 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T20:51:29,580 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1addd255-3f95-cd8c-0201-6dd0998cd993/hadoop.log.dir/,STOPPED} 2024-11-11T20:51:29,595 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T20:51:29,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down