2024-11-12 11:52:21,858 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-12 11:52:21,870 main DEBUG Took 0.010446 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 11:52:21,871 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 11:52:21,871 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 11:52:21,872 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 11:52:21,873 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,881 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 11:52:21,893 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,895 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,895 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,895 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,896 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,897 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,898 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,899 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,899 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,900 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,901 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,901 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,902 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,903 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,903 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,904 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 11:52:21,904 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,904 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 11:52:21,906 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 11:52:21,907 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 11:52:21,908 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 11:52:21,909 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 11:52:21,910 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 11:52:21,910 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 11:52:21,917 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 11:52:21,920 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 11:52:21,921 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 11:52:21,922 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 11:52:21,922 main DEBUG createAppenders(={Console}) 2024-11-12 11:52:21,923 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-12 11:52:21,923 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-12 11:52:21,923 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-12 11:52:21,924 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 11:52:21,924 main DEBUG OutputStream closed 2024-11-12 11:52:21,924 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 11:52:21,925 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 11:52:21,925 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-12 11:52:21,990 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 11:52:21,992 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 11:52:21,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 11:52:21,993 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 11:52:21,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 11:52:21,994 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 11:52:21,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 11:52:21,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 11:52:21,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 11:52:21,995 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 11:52:21,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 11:52:21,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 11:52:21,996 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 11:52:21,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 11:52:21,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 11:52:21,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 11:52:21,997 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 11:52:21,998 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 11:52:22,000 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 11:52:22,000 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-12 11:52:22,001 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 11:52:22,001 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-12T11:52:22,013 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-12 11:52:22,016 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 11:52:22,016 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T11:52:22,236 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305 2024-11-12T11:52:22,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T11:52:22,274 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680, deleteOnExit=true 2024-11-12T11:52:22,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T11:52:22,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/test.cache.data in system properties and HBase conf 2024-11-12T11:52:22,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T11:52:22,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir in system properties and HBase conf 2024-11-12T11:52:22,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T11:52:22,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T11:52:22,281 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T11:52:22,387 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T11:52:22,482 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T11:52:22,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T11:52:22,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T11:52:22,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T11:52:22,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T11:52:22,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T11:52:22,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T11:52:22,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T11:52:22,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T11:52:22,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T11:52:22,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/nfs.dump.dir in system properties and HBase conf 2024-11-12T11:52:22,490 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/java.io.tmpdir in system properties and HBase conf 2024-11-12T11:52:22,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T11:52:22,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T11:52:22,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T11:52:23,417 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T11:52:23,488 INFO [Time-limited test {}] log.Log(170): Logging initialized @2271ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T11:52:23,553 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T11:52:23,609 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T11:52:23,630 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T11:52:23,630 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T11:52:23,632 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T11:52:23,643 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T11:52:23,646 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir/,AVAILABLE} 2024-11-12T11:52:23,647 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T11:52:23,813 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/java.io.tmpdir/jetty-localhost-45661-hadoop-hdfs-3_4_1-tests_jar-_-any-10781085737716596640/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T11:52:23,819 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:45661} 2024-11-12T11:52:23,819 INFO [Time-limited test {}] server.Server(415): Started @2602ms 2024-11-12T11:52:24,313 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T11:52:24,321 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T11:52:24,322 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T11:52:24,323 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T11:52:24,323 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T11:52:24,324 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir/,AVAILABLE} 2024-11-12T11:52:24,324 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T11:52:24,424 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/java.io.tmpdir/jetty-localhost-36831-hadoop-hdfs-3_4_1-tests_jar-_-any-5816004617234905260/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T11:52:24,425 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:36831} 2024-11-12T11:52:24,425 INFO [Time-limited test {}] server.Server(415): Started @3208ms 2024-11-12T11:52:24,474 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T11:52:24,581 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T11:52:24,588 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T11:52:24,593 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T11:52:24,593 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T11:52:24,593 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T11:52:24,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir/,AVAILABLE} 2024-11-12T11:52:24,595 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T11:52:24,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/java.io.tmpdir/jetty-localhost-40289-hadoop-hdfs-3_4_1-tests_jar-_-any-5812215059886383047/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T11:52:24,722 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:40289} 2024-11-12T11:52:24,722 INFO [Time-limited test {}] server.Server(415): Started @3505ms 2024-11-12T11:52:24,725 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T11:52:24,763 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T11:52:24,768 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T11:52:24,770 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T11:52:24,770 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T11:52:24,770 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T11:52:24,771 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir/,AVAILABLE} 2024-11-12T11:52:24,772 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T11:52:24,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/java.io.tmpdir/jetty-localhost-43429-hadoop-hdfs-3_4_1-tests_jar-_-any-11526873755640853297/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T11:52:24,871 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:43429} 2024-11-12T11:52:24,872 INFO [Time-limited test {}] server.Server(415): Started @3654ms 2024-11-12T11:52:24,873 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T11:52:25,674 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data4/current/BP-1490150824-172.17.0.2-1731412343017/current, will proceed with Du for space computation calculation, 2024-11-12T11:52:25,674 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data2/current/BP-1490150824-172.17.0.2-1731412343017/current, will proceed with Du for space computation calculation, 2024-11-12T11:52:25,674 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data3/current/BP-1490150824-172.17.0.2-1731412343017/current, will proceed with Du for space computation calculation, 2024-11-12T11:52:25,674 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data1/current/BP-1490150824-172.17.0.2-1731412343017/current, will proceed with Du for space computation calculation, 2024-11-12T11:52:25,707 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T11:52:25,707 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T11:52:25,749 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x687d39cb2fe6af58 with lease ID 0x769b5acb15e925fc: Processing first storage report for DS-452caef6-73c2-4535-8909-6cc1964991cc from datanode DatanodeRegistration(127.0.0.1:39411, datanodeUuid=1fc17cea-dfae-4682-9c7c-d59a9b861595, infoPort=40641, infoSecurePort=0, ipcPort=34921, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017) 2024-11-12T11:52:25,750 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x687d39cb2fe6af58 with lease ID 0x769b5acb15e925fc: from storage DS-452caef6-73c2-4535-8909-6cc1964991cc node DatanodeRegistration(127.0.0.1:39411, datanodeUuid=1fc17cea-dfae-4682-9c7c-d59a9b861595, infoPort=40641, infoSecurePort=0, ipcPort=34921, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T11:52:25,751 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x751f15a1fde3bcb6 with lease ID 0x769b5acb15e925fb: Processing first storage report for DS-8937733f-a602-4fa6-8252-e0ec7163a924 from datanode DatanodeRegistration(127.0.0.1:42055, datanodeUuid=8dfd5a13-6277-405e-af3b-99df7f0e8c31, infoPort=33001, infoSecurePort=0, ipcPort=42559, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017) 2024-11-12T11:52:25,751 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x751f15a1fde3bcb6 with lease ID 0x769b5acb15e925fb: from storage DS-8937733f-a602-4fa6-8252-e0ec7163a924 node DatanodeRegistration(127.0.0.1:42055, datanodeUuid=8dfd5a13-6277-405e-af3b-99df7f0e8c31, infoPort=33001, infoSecurePort=0, ipcPort=42559, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T11:52:25,751 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x687d39cb2fe6af58 with lease ID 0x769b5acb15e925fc: Processing first storage report for DS-1fd69f37-8034-4ec7-aeb6-ed036d6514aa from datanode DatanodeRegistration(127.0.0.1:39411, datanodeUuid=1fc17cea-dfae-4682-9c7c-d59a9b861595, infoPort=40641, infoSecurePort=0, ipcPort=34921, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017) 2024-11-12T11:52:25,751 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x687d39cb2fe6af58 with lease ID 0x769b5acb15e925fc: from storage DS-1fd69f37-8034-4ec7-aeb6-ed036d6514aa node DatanodeRegistration(127.0.0.1:39411, datanodeUuid=1fc17cea-dfae-4682-9c7c-d59a9b861595, infoPort=40641, infoSecurePort=0, ipcPort=34921, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T11:52:25,751 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x751f15a1fde3bcb6 with lease ID 0x769b5acb15e925fb: Processing first storage report for DS-c96371ca-bc01-487f-8134-0c4f822ec84f from datanode DatanodeRegistration(127.0.0.1:42055, datanodeUuid=8dfd5a13-6277-405e-af3b-99df7f0e8c31, infoPort=33001, infoSecurePort=0, ipcPort=42559, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017) 2024-11-12T11:52:25,752 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x751f15a1fde3bcb6 with lease ID 0x769b5acb15e925fb: from storage DS-c96371ca-bc01-487f-8134-0c4f822ec84f node DatanodeRegistration(127.0.0.1:42055, datanodeUuid=8dfd5a13-6277-405e-af3b-99df7f0e8c31, infoPort=33001, infoSecurePort=0, ipcPort=42559, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T11:52:25,851 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data5/current/BP-1490150824-172.17.0.2-1731412343017/current, will proceed with Du for space computation calculation, 2024-11-12T11:52:25,851 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data6/current/BP-1490150824-172.17.0.2-1731412343017/current, will proceed with Du for space computation calculation, 2024-11-12T11:52:25,868 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T11:52:25,872 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4612dbade1e29dab with lease ID 0x769b5acb15e925fd: Processing first storage report for DS-2f88c530-085d-4189-a3bb-e2e8f41f6ab8 from datanode DatanodeRegistration(127.0.0.1:44441, datanodeUuid=16c2faf6-3c7c-4efd-823f-54c628db37f6, infoPort=42399, infoSecurePort=0, ipcPort=34469, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017) 2024-11-12T11:52:25,872 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4612dbade1e29dab with lease ID 0x769b5acb15e925fd: from storage DS-2f88c530-085d-4189-a3bb-e2e8f41f6ab8 node DatanodeRegistration(127.0.0.1:44441, datanodeUuid=16c2faf6-3c7c-4efd-823f-54c628db37f6, infoPort=42399, infoSecurePort=0, ipcPort=34469, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T11:52:25,872 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4612dbade1e29dab with lease ID 0x769b5acb15e925fd: Processing first storage report for DS-2b5a7cfa-f589-4598-8642-1fd1852ad86a from datanode DatanodeRegistration(127.0.0.1:44441, datanodeUuid=16c2faf6-3c7c-4efd-823f-54c628db37f6, infoPort=42399, infoSecurePort=0, ipcPort=34469, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017) 2024-11-12T11:52:25,873 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4612dbade1e29dab with lease ID 0x769b5acb15e925fd: from storage DS-2b5a7cfa-f589-4598-8642-1fd1852ad86a node DatanodeRegistration(127.0.0.1:44441, datanodeUuid=16c2faf6-3c7c-4efd-823f-54c628db37f6, infoPort=42399, infoSecurePort=0, ipcPort=34469, storageInfo=lv=-57;cid=testClusterID;nsid=1475437463;c=1731412343017), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T11:52:25,886 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305 2024-11-12T11:52:25,949 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/zookeeper_0, clientPort=60801, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T11:52:25,956 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60801 2024-11-12T11:52:25,966 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:25,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:26,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741825_1001 (size=7) 2024-11-12T11:52:26,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741825_1001 (size=7) 2024-11-12T11:52:26,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741825_1001 (size=7) 2024-11-12T11:52:26,589 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 with version=8 2024-11-12T11:52:26,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/hbase-staging 2024-11-12T11:52:26,673 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T11:52:26,894 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b51f50ce4a7c:0 server-side Connection retries=45 2024-11-12T11:52:26,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:26,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:26,908 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T11:52:26,908 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:26,908 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T11:52:27,033 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T11:52:27,083 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T11:52:27,091 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T11:52:27,094 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T11:52:27,116 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32362 (auto-detected) 2024-11-12T11:52:27,117 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T11:52:27,133 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37827 2024-11-12T11:52:27,152 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37827 connecting to ZooKeeper ensemble=127.0.0.1:60801 2024-11-12T11:52:27,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:378270x0, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T11:52:27,269 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37827-0x101302d66c30000 connected 2024-11-12T11:52:27,348 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,353 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T11:52:27,369 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09, hbase.cluster.distributed=false 2024-11-12T11:52:27,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T11:52:27,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37827 2024-11-12T11:52:27,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37827 2024-11-12T11:52:27,399 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37827 2024-11-12T11:52:27,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37827 2024-11-12T11:52:27,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37827 2024-11-12T11:52:27,496 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b51f50ce4a7c:0 server-side Connection retries=45 2024-11-12T11:52:27,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,498 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T11:52:27,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T11:52:27,501 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T11:52:27,503 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T11:52:27,504 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44417 2024-11-12T11:52:27,506 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44417 connecting to ZooKeeper ensemble=127.0.0.1:60801 2024-11-12T11:52:27,507 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,509 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444170x0, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T11:52:27,522 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:444170x0, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T11:52:27,522 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44417-0x101302d66c30001 connected 2024-11-12T11:52:27,525 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T11:52:27,532 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T11:52:27,535 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T11:52:27,540 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T11:52:27,541 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44417 2024-11-12T11:52:27,541 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44417 2024-11-12T11:52:27,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44417 2024-11-12T11:52:27,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44417 2024-11-12T11:52:27,542 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44417 2024-11-12T11:52:27,558 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b51f50ce4a7c:0 server-side Connection retries=45 2024-11-12T11:52:27,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,559 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T11:52:27,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T11:52:27,559 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T11:52:27,559 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T11:52:27,560 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38813 2024-11-12T11:52:27,562 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38813 connecting to ZooKeeper ensemble=127.0.0.1:60801 2024-11-12T11:52:27,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,564 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388130x0, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T11:52:27,577 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T11:52:27,577 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38813-0x101302d66c30002 connected 2024-11-12T11:52:27,577 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T11:52:27,578 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T11:52:27,580 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T11:52:27,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T11:52:27,586 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-12T11:52:27,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38813 2024-11-12T11:52:27,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38813 2024-11-12T11:52:27,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-12T11:52:27,588 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-12T11:52:27,604 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b51f50ce4a7c:0 server-side Connection retries=45 2024-11-12T11:52:27,604 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,605 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T11:52:27,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T11:52:27,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T11:52:27,605 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T11:52:27,605 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T11:52:27,606 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34963 2024-11-12T11:52:27,608 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34963 connecting to ZooKeeper ensemble=127.0.0.1:60801 2024-11-12T11:52:27,609 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,611 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349630x0, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T11:52:27,622 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:349630x0, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T11:52:27,622 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34963-0x101302d66c30003 connected 2024-11-12T11:52:27,622 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T11:52:27,624 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T11:52:27,625 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T11:52:27,627 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T11:52:27,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34963 2024-11-12T11:52:27,628 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34963 2024-11-12T11:52:27,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34963 2024-11-12T11:52:27,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34963 2024-11-12T11:52:27,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34963 2024-11-12T11:52:27,651 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b51f50ce4a7c:37827 2024-11-12T11:52:27,652 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:27,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T11:52:27,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T11:52:27,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T11:52:27,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T11:52:27,670 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:27,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T11:52:27,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T11:52:27,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T11:52:27,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:27,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:27,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:27,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:27,698 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T11:52:27,699 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b51f50ce4a7c,37827,1731412346745 from backup master directory 2024-11-12T11:52:27,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:27,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T11:52:27,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T11:52:27,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T11:52:27,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T11:52:27,710 WARN [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T11:52:27,711 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:27,713 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T11:52:27,715 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T11:52:27,766 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/hbase.id] with ID: c65935c0-1a43-48b1-b034-8244d9026a63 2024-11-12T11:52:27,767 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/.tmp/hbase.id 2024-11-12T11:52:27,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741826_1002 (size=42) 2024-11-12T11:52:27,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741826_1002 (size=42) 2024-11-12T11:52:27,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741826_1002 (size=42) 2024-11-12T11:52:27,784 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/.tmp/hbase.id]:[hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/hbase.id] 2024-11-12T11:52:27,825 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T11:52:27,830 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T11:52:27,849 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-12T11:52:27,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:27,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:27,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:27,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:27,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741827_1003 (size=196) 2024-11-12T11:52:27,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741827_1003 (size=196) 2024-11-12T11:52:27,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741827_1003 (size=196) 2024-11-12T11:52:27,894 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T11:52:27,896 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T11:52:27,909 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T11:52:27,913 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T11:52:27,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741828_1004 (size=1189) 2024-11-12T11:52:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741828_1004 (size=1189) 2024-11-12T11:52:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741828_1004 (size=1189) 2024-11-12T11:52:27,959 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store 2024-11-12T11:52:27,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741829_1005 (size=34) 2024-11-12T11:52:27,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741829_1005 (size=34) 2024-11-12T11:52:27,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741829_1005 (size=34) 2024-11-12T11:52:27,987 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T11:52:27,990 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:27,991 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T11:52:27,991 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T11:52:27,991 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T11:52:27,993 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T11:52:27,993 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T11:52:27,993 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T11:52:27,994 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731412347991Disabling compacts and flushes for region at 1731412347991Disabling writes for close at 1731412347993 (+2 ms)Writing region close event to WAL at 1731412347993Closed at 1731412347993 2024-11-12T11:52:27,996 WARN [master/b51f50ce4a7c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/.initializing 2024-11-12T11:52:27,996 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/WALs/b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:28,004 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T11:52:28,018 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b51f50ce4a7c%2C37827%2C1731412346745, suffix=, logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/WALs/b51f50ce4a7c,37827,1731412346745, archiveDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/oldWALs, maxLogs=10 2024-11-12T11:52:28,039 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/WALs/b51f50ce4a7c,37827,1731412346745/b51f50ce4a7c%2C37827%2C1731412346745.1731412348022, exclude list is [], retry=0 2024-11-12T11:52:28,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44441,DS-2f88c530-085d-4189-a3bb-e2e8f41f6ab8,DISK] 2024-11-12T11:52:28,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42055,DS-8937733f-a602-4fa6-8252-e0ec7163a924,DISK] 2024-11-12T11:52:28,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39411,DS-452caef6-73c2-4535-8909-6cc1964991cc,DISK] 2024-11-12T11:52:28,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T11:52:28,097 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/WALs/b51f50ce4a7c,37827,1731412346745/b51f50ce4a7c%2C37827%2C1731412346745.1731412348022 2024-11-12T11:52:28,099 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33001:33001),(127.0.0.1/127.0.0.1:40641:40641),(127.0.0.1/127.0.0.1:42399:42399)] 2024-11-12T11:52:28,100 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T11:52:28,100 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:28,104 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,106 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T11:52:28,170 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:28,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,176 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T11:52:28,177 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:28,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T11:52:28,181 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:28,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T11:52:28,187 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:28,188 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,191 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,193 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,198 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,199 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,202 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T11:52:28,206 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T11:52:28,211 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:28,213 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70906310, jitterRate=0.05658635497093201}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T11:52:28,222 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731412348117Initializing all the Stores at 1731412348120 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412348120Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412348121 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412348121Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412348121Cleaning up temporary data from old regions at 1731412348199 (+78 ms)Region opened successfully at 1731412348222 (+23 ms) 2024-11-12T11:52:28,224 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T11:52:28,268 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@266c6f27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b51f50ce4a7c/172.17.0.2:0 2024-11-12T11:52:28,299 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T11:52:28,309 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:28,309 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:28,312 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:28,313 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T11:52:28,317 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T11:52:28,318 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T11:52:28,344 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T11:52:28,351 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T11:52:28,396 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T11:52:28,401 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T11:52:28,404 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T11:52:28,413 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T11:52:28,415 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T11:52:28,419 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T11:52:28,429 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T11:52:28,431 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T11:52:28,442 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T11:52:28,464 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T11:52:28,475 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T11:52:28,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T11:52:28,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T11:52:28,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T11:52:28,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T11:52:28,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,493 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b51f50ce4a7c,37827,1731412346745, sessionid=0x101302d66c30000, setting cluster-up flag (Was=false) 2024-11-12T11:52:28,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,546 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T11:52:28,550 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:28,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:28,601 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T11:52:28,604 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:28,614 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T11:52:28,635 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(746): ClusterId : c65935c0-1a43-48b1-b034-8244d9026a63 2024-11-12T11:52:28,635 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(746): ClusterId : c65935c0-1a43-48b1-b034-8244d9026a63 2024-11-12T11:52:28,635 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(746): ClusterId : c65935c0-1a43-48b1-b034-8244d9026a63 2024-11-12T11:52:28,638 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T11:52:28,638 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T11:52:28,638 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T11:52:28,648 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T11:52:28,648 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T11:52:28,648 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T11:52:28,648 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T11:52:28,648 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T11:52:28,648 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T11:52:28,668 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T11:52:28,668 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T11:52:28,668 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T11:52:28,669 DEBUG [RS:0;b51f50ce4a7c:44417 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2001d090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b51f50ce4a7c/172.17.0.2:0 2024-11-12T11:52:28,669 DEBUG [RS:1;b51f50ce4a7c:38813 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48382ce5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b51f50ce4a7c/172.17.0.2:0 2024-11-12T11:52:28,669 DEBUG [RS:2;b51f50ce4a7c:34963 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c91f9d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b51f50ce4a7c/172.17.0.2:0 2024-11-12T11:52:28,684 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b51f50ce4a7c:34963 2024-11-12T11:52:28,685 DEBUG [RS:1;b51f50ce4a7c:38813 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b51f50ce4a7c:38813 2024-11-12T11:52:28,686 DEBUG [RS:0;b51f50ce4a7c:44417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b51f50ce4a7c:44417 2024-11-12T11:52:28,687 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T11:52:28,687 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T11:52:28,687 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T11:52:28,687 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T11:52:28,687 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T11:52:28,687 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T11:52:28,687 DEBUG [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T11:52:28,687 DEBUG [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T11:52:28,687 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T11:52:28,689 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(2659): reportForDuty to master=b51f50ce4a7c,37827,1731412346745 with port=34963, startcode=1731412347604 2024-11-12T11:52:28,689 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(2659): reportForDuty to master=b51f50ce4a7c,37827,1731412346745 with port=44417, startcode=1731412347465 2024-11-12T11:52:28,689 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(2659): reportForDuty to master=b51f50ce4a7c,37827,1731412346745 with port=38813, startcode=1731412347557 2024-11-12T11:52:28,689 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:28,701 DEBUG [RS:0;b51f50ce4a7c:44417 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T11:52:28,701 DEBUG [RS:2;b51f50ce4a7c:34963 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T11:52:28,701 DEBUG [RS:1;b51f50ce4a7c:38813 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T11:52:28,701 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T11:52:28,709 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T11:52:28,716 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b51f50ce4a7c,37827,1731412346745 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T11:52:28,725 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b51f50ce4a7c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T11:52:28,726 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b51f50ce4a7c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T11:52:28,727 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b51f50ce4a7c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T11:52:28,727 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b51f50ce4a7c:0, corePoolSize=5, maxPoolSize=5 2024-11-12T11:52:28,727 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b51f50ce4a7c:0, corePoolSize=10, maxPoolSize=10 2024-11-12T11:52:28,728 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:28,728 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b51f50ce4a7c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T11:52:28,728 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:28,735 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412378735 2024-11-12T11:52:28,738 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T11:52:28,739 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T11:52:28,739 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39843, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T11:52:28,739 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T11:52:28,740 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58159, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T11:52:28,743 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:28,743 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T11:52:28,745 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T11:52:28,745 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T11:52:28,746 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T11:52:28,746 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T11:52:28,747 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:28,747 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:28,750 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,751 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T11:52:28,751 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T11:52:28,752 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T11:52:28,753 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T11:52:28,753 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:28,754 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:28,756 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T11:52:28,756 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T11:52:28,762 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b51f50ce4a7c:0:becomeActiveMaster-HFileCleaner.large.0-1731412348758,5,FailOnTimeoutGroup] 2024-11-12T11:52:28,762 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b51f50ce4a7c:0:becomeActiveMaster-HFileCleaner.small.0-1731412348762,5,FailOnTimeoutGroup] 2024-11-12T11:52:28,763 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:28,763 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T11:52:28,764 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:28,764 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:28,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741831_1007 (size=1321) 2024-11-12T11:52:28,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741831_1007 (size=1321) 2024-11-12T11:52:28,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741831_1007 (size=1321) 2024-11-12T11:52:28,800 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T11:52:28,800 DEBUG [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T11:52:28,800 DEBUG [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T11:52:28,800 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T11:52:28,800 WARN [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T11:52:28,800 WARN [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T11:52:28,800 WARN [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T11:52:28,800 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:28,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741832_1008 (size=32) 2024-11-12T11:52:28,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741832_1008 (size=32) 2024-11-12T11:52:28,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741832_1008 (size=32) 2024-11-12T11:52:28,814 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:28,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T11:52:28,819 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T11:52:28,819 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:28,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T11:52:28,822 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T11:52:28,822 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:28,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T11:52:28,826 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T11:52:28,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:28,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T11:52:28,829 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T11:52:28,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:28,830 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:28,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T11:52:28,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740 2024-11-12T11:52:28,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740 2024-11-12T11:52:28,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T11:52:28,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T11:52:28,837 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T11:52:28,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T11:52:28,842 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:28,843 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68836808, jitterRate=0.025748372077941895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T11:52:28,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731412348815Initializing all the Stores at 1731412348816 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412348816Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412348816Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412348816Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412348816Cleaning up temporary data from old regions at 1731412348836 (+20 ms)Region opened successfully at 1731412348847 (+11 ms) 2024-11-12T11:52:28,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T11:52:28,847 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T11:52:28,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T11:52:28,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T11:52:28,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T11:52:28,848 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T11:52:28,849 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731412348847Disabling compacts and flushes for region at 1731412348847Disabling writes for close at 1731412348847Writing region close event to WAL at 1731412348848 (+1 ms)Closed at 1731412348848 2024-11-12T11:52:28,852 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:28,852 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T11:52:28,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T11:52:28,867 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:28,870 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T11:52:28,902 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(2659): reportForDuty to master=b51f50ce4a7c,37827,1731412346745 with port=34963, startcode=1731412347604 2024-11-12T11:52:28,902 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(2659): reportForDuty to master=b51f50ce4a7c,37827,1731412346745 with port=44417, startcode=1731412347465 2024-11-12T11:52:28,902 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(2659): reportForDuty to master=b51f50ce4a7c,37827,1731412346745 with port=38813, startcode=1731412347557 2024-11-12T11:52:28,906 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:28,910 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] master.ServerManager(517): Registering regionserver=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:28,920 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:28,921 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] master.ServerManager(517): Registering regionserver=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:28,921 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:28,921 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42603 2024-11-12T11:52:28,921 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T11:52:28,923 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:28,923 DEBUG [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:28,924 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] master.ServerManager(517): Registering regionserver=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:28,924 DEBUG [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42603 2024-11-12T11:52:28,924 DEBUG [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T11:52:28,926 DEBUG [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:28,926 DEBUG [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42603 2024-11-12T11:52:28,926 DEBUG [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T11:52:28,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T11:52:28,968 DEBUG [RS:2;b51f50ce4a7c:34963 {}] zookeeper.ZKUtil(111): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:28,968 DEBUG [RS:1;b51f50ce4a7c:38813 {}] zookeeper.ZKUtil(111): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:28,968 WARN [RS:1;b51f50ce4a7c:38813 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T11:52:28,968 WARN [RS:2;b51f50ce4a7c:34963 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T11:52:28,968 INFO [RS:1;b51f50ce4a7c:38813 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T11:52:28,968 INFO [RS:2;b51f50ce4a7c:34963 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T11:52:28,969 DEBUG [RS:0;b51f50ce4a7c:44417 {}] zookeeper.ZKUtil(111): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:28,969 WARN [RS:0;b51f50ce4a7c:44417 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T11:52:28,969 DEBUG [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:28,969 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:28,969 INFO [RS:0;b51f50ce4a7c:44417 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T11:52:28,970 DEBUG [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:28,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b51f50ce4a7c,44417,1731412347465] 2024-11-12T11:52:28,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b51f50ce4a7c,38813,1731412347557] 2024-11-12T11:52:28,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b51f50ce4a7c,34963,1731412347604] 2024-11-12T11:52:28,996 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T11:52:28,996 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T11:52:28,996 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T11:52:29,013 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T11:52:29,013 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T11:52:29,013 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T11:52:29,018 INFO [RS:2;b51f50ce4a7c:34963 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T11:52:29,018 INFO [RS:1;b51f50ce4a7c:38813 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T11:52:29,018 INFO [RS:0;b51f50ce4a7c:44417 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T11:52:29,018 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,018 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,018 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,019 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T11:52:29,019 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T11:52:29,019 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T11:52:29,021 WARN [b51f50ce4a7c:37827 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T11:52:29,025 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T11:52:29,025 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T11:52:29,025 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T11:52:29,026 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,026 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,026 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T11:52:29,027 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T11:52:29,027 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0, corePoolSize=2, maxPoolSize=2 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,027 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b51f50ce4a7c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T11:52:29,028 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b51f50ce4a7c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T11:52:29,028 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b51f50ce4a7c:0, corePoolSize=1, maxPoolSize=1 2024-11-12T11:52:29,028 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b51f50ce4a7c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T11:52:29,028 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b51f50ce4a7c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T11:52:29,028 DEBUG [RS:1;b51f50ce4a7c:38813 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b51f50ce4a7c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T11:52:29,028 DEBUG [RS:0;b51f50ce4a7c:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b51f50ce4a7c:0, corePoolSize=3, maxPoolSize=3 2024-11-12T11:52:29,031 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,34963,1731412347604-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,031 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,032 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,032 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,38813,1731412347557-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T11:52:29,032 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,032 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,032 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,032 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,032 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,032 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,44417,1731412347465-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T11:52:29,049 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T11:52:29,049 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T11:52:29,051 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,44417,1731412347465-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,051 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,38813,1731412347557-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,051 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,051 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,051 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.Replication(171): b51f50ce4a7c,44417,1731412347465 started 2024-11-12T11:52:29,051 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.Replication(171): b51f50ce4a7c,38813,1731412347557 started 2024-11-12T11:52:29,051 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T11:52:29,052 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,34963,1731412347604-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,052 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,052 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.Replication(171): b51f50ce4a7c,34963,1731412347604 started 2024-11-12T11:52:29,068 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,068 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,068 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(1482): Serving as b51f50ce4a7c,44417,1731412347465, RpcServer on b51f50ce4a7c/172.17.0.2:44417, sessionid=0x101302d66c30001 2024-11-12T11:52:29,068 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(1482): Serving as b51f50ce4a7c,38813,1731412347557, RpcServer on b51f50ce4a7c/172.17.0.2:38813, sessionid=0x101302d66c30002 2024-11-12T11:52:29,069 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T11:52:29,069 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T11:52:29,069 DEBUG [RS:0;b51f50ce4a7c:44417 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:29,069 DEBUG [RS:1;b51f50ce4a7c:38813 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:29,069 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b51f50ce4a7c,44417,1731412347465' 2024-11-12T11:52:29,069 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b51f50ce4a7c,38813,1731412347557' 2024-11-12T11:52:29,069 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T11:52:29,069 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T11:52:29,070 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T11:52:29,070 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T11:52:29,070 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T11:52:29,070 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T11:52:29,070 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T11:52:29,070 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T11:52:29,071 DEBUG [RS:0;b51f50ce4a7c:44417 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:29,071 DEBUG [RS:1;b51f50ce4a7c:38813 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:29,071 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b51f50ce4a7c,44417,1731412347465' 2024-11-12T11:52:29,071 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b51f50ce4a7c,38813,1731412347557' 2024-11-12T11:52:29,071 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T11:52:29,071 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T11:52:29,071 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T11:52:29,071 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T11:52:29,072 DEBUG [RS:0;b51f50ce4a7c:44417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T11:52:29,072 DEBUG [RS:1;b51f50ce4a7c:38813 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T11:52:29,072 INFO [RS:1;b51f50ce4a7c:38813 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T11:52:29,072 INFO [RS:0;b51f50ce4a7c:44417 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T11:52:29,072 INFO [RS:0;b51f50ce4a7c:44417 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T11:52:29,072 INFO [RS:1;b51f50ce4a7c:38813 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T11:52:29,073 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,073 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1482): Serving as b51f50ce4a7c,34963,1731412347604, RpcServer on b51f50ce4a7c/172.17.0.2:34963, sessionid=0x101302d66c30003 2024-11-12T11:52:29,073 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T11:52:29,074 DEBUG [RS:2;b51f50ce4a7c:34963 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:29,074 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b51f50ce4a7c,34963,1731412347604' 2024-11-12T11:52:29,074 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T11:52:29,074 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T11:52:29,075 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T11:52:29,075 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T11:52:29,075 DEBUG [RS:2;b51f50ce4a7c:34963 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:29,075 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b51f50ce4a7c,34963,1731412347604' 2024-11-12T11:52:29,075 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T11:52:29,076 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T11:52:29,076 DEBUG [RS:2;b51f50ce4a7c:34963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T11:52:29,076 INFO [RS:2;b51f50ce4a7c:34963 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T11:52:29,077 INFO [RS:2;b51f50ce4a7c:34963 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T11:52:29,179 INFO [RS:0;b51f50ce4a7c:44417 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T11:52:29,179 INFO [RS:2;b51f50ce4a7c:34963 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T11:52:29,179 INFO [RS:1;b51f50ce4a7c:38813 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T11:52:29,183 INFO [RS:0;b51f50ce4a7c:44417 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b51f50ce4a7c%2C44417%2C1731412347465, suffix=, logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,44417,1731412347465, archiveDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/oldWALs, maxLogs=32 2024-11-12T11:52:29,183 INFO [RS:1;b51f50ce4a7c:38813 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b51f50ce4a7c%2C38813%2C1731412347557, suffix=, logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557, archiveDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/oldWALs, maxLogs=32 2024-11-12T11:52:29,183 INFO [RS:2;b51f50ce4a7c:34963 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b51f50ce4a7c%2C34963%2C1731412347604, suffix=, logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,34963,1731412347604, archiveDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/oldWALs, maxLogs=32 2024-11-12T11:52:29,199 DEBUG [RS:0;b51f50ce4a7c:44417 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,44417,1731412347465/b51f50ce4a7c%2C44417%2C1731412347465.1731412349187, exclude list is [], retry=0 2024-11-12T11:52:29,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42055,DS-8937733f-a602-4fa6-8252-e0ec7163a924,DISK] 2024-11-12T11:52:29,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39411,DS-452caef6-73c2-4535-8909-6cc1964991cc,DISK] 2024-11-12T11:52:29,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44441,DS-2f88c530-085d-4189-a3bb-e2e8f41f6ab8,DISK] 2024-11-12T11:52:29,206 DEBUG [RS:1;b51f50ce4a7c:38813 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, exclude list is [], retry=0 2024-11-12T11:52:29,206 DEBUG [RS:2;b51f50ce4a7c:34963 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,34963,1731412347604/b51f50ce4a7c%2C34963%2C1731412347604.1731412349187, exclude list is [], retry=0 2024-11-12T11:52:29,208 INFO [RS:0;b51f50ce4a7c:44417 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,44417,1731412347465/b51f50ce4a7c%2C44417%2C1731412347465.1731412349187 2024-11-12T11:52:29,211 DEBUG [RS:0;b51f50ce4a7c:44417 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33001:33001),(127.0.0.1/127.0.0.1:42399:42399),(127.0.0.1/127.0.0.1:40641:40641)] 2024-11-12T11:52:29,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39411,DS-452caef6-73c2-4535-8909-6cc1964991cc,DISK] 2024-11-12T11:52:29,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42055,DS-8937733f-a602-4fa6-8252-e0ec7163a924,DISK] 2024-11-12T11:52:29,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39411,DS-452caef6-73c2-4535-8909-6cc1964991cc,DISK] 2024-11-12T11:52:29,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42055,DS-8937733f-a602-4fa6-8252-e0ec7163a924,DISK] 2024-11-12T11:52:29,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44441,DS-2f88c530-085d-4189-a3bb-e2e8f41f6ab8,DISK] 2024-11-12T11:52:29,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44441,DS-2f88c530-085d-4189-a3bb-e2e8f41f6ab8,DISK] 2024-11-12T11:52:29,217 INFO [RS:1;b51f50ce4a7c:38813 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187 2024-11-12T11:52:29,218 DEBUG [RS:1;b51f50ce4a7c:38813 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40641:40641),(127.0.0.1/127.0.0.1:42399:42399),(127.0.0.1/127.0.0.1:33001:33001)] 2024-11-12T11:52:29,218 INFO [RS:2;b51f50ce4a7c:34963 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,34963,1731412347604/b51f50ce4a7c%2C34963%2C1731412347604.1731412349187 2024-11-12T11:52:29,218 DEBUG [RS:2;b51f50ce4a7c:34963 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40641:40641),(127.0.0.1/127.0.0.1:33001:33001),(127.0.0.1/127.0.0.1:42399:42399)] 2024-11-12T11:52:29,274 DEBUG [b51f50ce4a7c:37827 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-12T11:52:29,283 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(204): Hosts are {b51f50ce4a7c=0} racks are {/default-rack=0} 2024-11-12T11:52:29,290 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T11:52:29,290 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T11:52:29,290 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T11:52:29,290 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T11:52:29,290 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T11:52:29,290 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T11:52:29,290 INFO [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T11:52:29,290 INFO [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T11:52:29,290 INFO [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T11:52:29,290 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T11:52:29,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:29,302 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,38813,1731412347557, state=OPENING 2024-11-12T11:52:29,312 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T11:52:29,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:29,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:29,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:29,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:29,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:29,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:29,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:29,323 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:29,325 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:29,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:29,507 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T11:52:29,509 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42639, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T11:52:29,522 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T11:52:29,522 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T11:52:29,523 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T11:52:29,526 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b51f50ce4a7c%2C38813%2C1731412347557.meta, suffix=.meta, logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557, archiveDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/oldWALs, maxLogs=32 2024-11-12T11:52:29,548 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, exclude list is [], retry=0 2024-11-12T11:52:29,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39411,DS-452caef6-73c2-4535-8909-6cc1964991cc,DISK] 2024-11-12T11:52:29,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44441,DS-2f88c530-085d-4189-a3bb-e2e8f41f6ab8,DISK] 2024-11-12T11:52:29,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42055,DS-8937733f-a602-4fa6-8252-e0ec7163a924,DISK] 2024-11-12T11:52:29,555 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta 2024-11-12T11:52:29,556 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42399:42399),(127.0.0.1/127.0.0.1:40641:40641),(127.0.0.1/127.0.0.1:33001:33001)] 2024-11-12T11:52:29,556 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T11:52:29,558 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T11:52:29,560 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T11:52:29,564 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T11:52:29,568 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T11:52:29,568 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:29,569 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T11:52:29,569 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T11:52:29,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T11:52:29,573 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T11:52:29,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:29,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:29,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T11:52:29,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T11:52:29,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:29,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:29,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T11:52:29,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T11:52:29,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:29,578 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:29,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T11:52:29,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T11:52:29,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:29,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:29,581 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T11:52:29,583 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740 2024-11-12T11:52:29,585 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740 2024-11-12T11:52:29,588 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T11:52:29,588 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T11:52:29,589 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T11:52:29,591 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T11:52:29,593 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67407234, jitterRate=0.004446059465408325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T11:52:29,593 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T11:52:29,594 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731412349569Writing region info on filesystem at 1731412349569Initializing all the Stores at 1731412349571 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412349571Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412349571Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412349571Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412349571Cleaning up temporary data from old regions at 1731412349588 (+17 ms)Running coprocessor post-open hooks at 1731412349593 (+5 ms)Region opened successfully at 1731412349594 (+1 ms) 2024-11-12T11:52:29,601 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731412349499 2024-11-12T11:52:29,610 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T11:52:29,611 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T11:52:29,612 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:29,614 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,38813,1731412347557, state=OPEN 2024-11-12T11:52:29,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:29,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:29,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:29,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:29,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:29,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:29,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:29,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:29,655 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:29,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T11:52:29,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,38813,1731412347557 in 329 msec 2024-11-12T11:52:29,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T11:52:29,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 806 msec 2024-11-12T11:52:29,669 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:29,669 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T11:52:29,686 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T11:52:29,687 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1] 2024-11-12T11:52:29,705 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T11:52:29,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42695, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T11:52:29,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0810 sec 2024-11-12T11:52:29,727 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731412349727, completionTime=-1 2024-11-12T11:52:29,730 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-12T11:52:29,730 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:29,755 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-12T11:52:29,755 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412409755 2024-11-12T11:52:29,755 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412469755 2024-11-12T11:52:29,755 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-12T11:52:29,757 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:29,763 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,37827,1731412346745-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,763 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,37827,1731412346745-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,764 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,37827,1731412346745-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,765 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b51f50ce4a7c:37827, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,765 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,766 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,771 DEBUG [master/b51f50ce4a7c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T11:52:29,791 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.080sec 2024-11-12T11:52:29,793 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T11:52:29,794 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T11:52:29,795 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T11:52:29,795 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T11:52:29,795 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T11:52:29,796 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,37827,1731412346745-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T11:52:29,796 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,37827,1731412346745-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T11:52:29,800 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T11:52:29,801 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T11:52:29,802 INFO [master/b51f50ce4a7c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b51f50ce4a7c,37827,1731412346745-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T11:52:29,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T11:52:29,849 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T11:52:29,849 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T11:52:29,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b51f50ce4a7c,37827,-1 for getting cluster id 2024-11-12T11:52:29,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T11:52:29,863 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c65935c0-1a43-48b1-b034-8244d9026a63' 2024-11-12T11:52:29,865 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T11:52:29,866 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c65935c0-1a43-48b1-b034-8244d9026a63" 2024-11-12T11:52:29,868 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@246a4f2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T11:52:29,868 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b51f50ce4a7c,37827,-1] 2024-11-12T11:52:29,870 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T11:52:29,872 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:29,873 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T11:52:29,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T11:52:29,877 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T11:52:29,885 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1] 2024-11-12T11:52:29,885 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T11:52:29,888 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T11:52:29,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:29,910 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T11:52:29,913 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:29,915 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@71c6a8cd 2024-11-12T11:52:29,916 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T11:52:29,918 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T11:52:29,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T11:52:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:29,962 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T11:52:29,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-12T11:52:29,964 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:29,968 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T11:52:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T11:52:29,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741837_1013 (size=358) 2024-11-12T11:52:29,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741837_1013 (size=358) 2024-11-12T11:52:29,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741837_1013 (size=358) 2024-11-12T11:52:29,998 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a0b1117abc263d25216a06f7640a6dd0, NAME => 'test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:29,998 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 57a12d2944d6f25f894da49363450447, NAME => 'test,ccc,1731412349920.57a12d2944d6f25f894da49363450447.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:29,999 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => cbfb2956e6def690689f5e33134c9dd4, NAME => 'test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741838_1014 (size=45) 2024-11-12T11:52:30,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741838_1014 (size=45) 2024-11-12T11:52:30,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741838_1014 (size=45) 2024-11-12T11:52:30,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing cbfb2956e6def690689f5e33134c9dd4, disabling compactions & flushes 2024-11-12T11:52:30,021 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:30,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:30,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. after waiting 0 ms 2024-11-12T11:52:30,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:30,021 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:30,021 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for cbfb2956e6def690689f5e33134c9dd4: Waiting for close lock at 1731412350021Disabling compacts and flushes for region at 1731412350021Disabling writes for close at 1731412350021Writing region close event to WAL at 1731412350021Closed at 1731412350021 2024-11-12T11:52:30,022 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f9e98aed23b0727af5f513c5de61a616, NAME => 'test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741839_1015 (size=42) 2024-11-12T11:52:30,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741840_1016 (size=45) 2024-11-12T11:52:30,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741840_1016 (size=45) 2024-11-12T11:52:30,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741839_1015 (size=42) 2024-11-12T11:52:30,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741840_1016 (size=45) 2024-11-12T11:52:30,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741839_1015 (size=42) 2024-11-12T11:52:30,031 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731412349920.57a12d2944d6f25f894da49363450447.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,031 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 57a12d2944d6f25f894da49363450447, disabling compactions & flushes 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a0b1117abc263d25216a06f7640a6dd0, disabling compactions & flushes 2024-11-12T11:52:30,032 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:30,032 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0. 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0. 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. after waiting 0 ms 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0. after waiting 0 ms 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0. 2024-11-12T11:52:30,032 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:30,032 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0. 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 57a12d2944d6f25f894da49363450447: Waiting for close lock at 1731412350032Disabling compacts and flushes for region at 1731412350032Disabling writes for close at 1731412350032Writing region close event to WAL at 1731412350032Closed at 1731412350032 2024-11-12T11:52:30,032 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a0b1117abc263d25216a06f7640a6dd0: Waiting for close lock at 1731412350032Disabling compacts and flushes for region at 1731412350032Disabling writes for close at 1731412350032Writing region close event to WAL at 1731412350032Closed at 1731412350032 2024-11-12T11:52:30,033 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f786dd883997cd82efafbe7d3d173fc7, NAME => 'test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,033 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 57a392190264a7b69a7d25f871aed2c7, NAME => 'test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741841_1017 (size=45) 2024-11-12T11:52:30,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741841_1017 (size=45) 2024-11-12T11:52:30,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741841_1017 (size=45) 2024-11-12T11:52:30,046 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,046 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f9e98aed23b0727af5f513c5de61a616, disabling compactions & flushes 2024-11-12T11:52:30,046 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:30,046 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:30,046 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. after waiting 0 ms 2024-11-12T11:52:30,046 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:30,046 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:30,046 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f9e98aed23b0727af5f513c5de61a616: Waiting for close lock at 1731412350046Disabling compacts and flushes for region at 1731412350046Disabling writes for close at 1731412350046Writing region close event to WAL at 1731412350046Closed at 1731412350046 2024-11-12T11:52:30,047 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => cf142f73f167aa11416e3ae10a6bf795, NAME => 'test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741843_1019 (size=45) 2024-11-12T11:52:30,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741843_1019 (size=45) 2024-11-12T11:52:30,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741843_1019 (size=45) 2024-11-12T11:52:30,055 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,055 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f786dd883997cd82efafbe7d3d173fc7, disabling compactions & flushes 2024-11-12T11:52:30,056 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:30,056 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741842_1018 (size=45) 2024-11-12T11:52:30,056 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. after waiting 0 ms 2024-11-12T11:52:30,056 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:30,056 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:30,056 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f786dd883997cd82efafbe7d3d173fc7: Waiting for close lock at 1731412350055Disabling compacts and flushes for region at 1731412350055Disabling writes for close at 1731412350056 (+1 ms)Writing region close event to WAL at 1731412350056Closed at 1731412350056 2024-11-12T11:52:30,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741842_1018 (size=45) 2024-11-12T11:52:30,057 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f3aaa68bd9553e5b9366c15cdfd85491, NAME => 'test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741842_1018 (size=45) 2024-11-12T11:52:30,060 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,061 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 57a392190264a7b69a7d25f871aed2c7, disabling compactions & flushes 2024-11-12T11:52:30,061 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. 2024-11-12T11:52:30,061 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. 2024-11-12T11:52:30,061 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. after waiting 0 ms 2024-11-12T11:52:30,061 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. 2024-11-12T11:52:30,061 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. 2024-11-12T11:52:30,061 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 57a392190264a7b69a7d25f871aed2c7: Waiting for close lock at 1731412350061Disabling compacts and flushes for region at 1731412350061Disabling writes for close at 1731412350061Writing region close event to WAL at 1731412350061Closed at 1731412350061 2024-11-12T11:52:30,062 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9ae3a40ca66d508699f89dac1ef676d7, NAME => 'test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741844_1020 (size=45) 2024-11-12T11:52:30,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741844_1020 (size=45) 2024-11-12T11:52:30,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741844_1020 (size=45) 2024-11-12T11:52:30,075 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,076 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing cf142f73f167aa11416e3ae10a6bf795, disabling compactions & flushes 2024-11-12T11:52:30,076 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:30,076 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:30,076 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. after waiting 0 ms 2024-11-12T11:52:30,076 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:30,076 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:30,076 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for cf142f73f167aa11416e3ae10a6bf795: Waiting for close lock at 1731412350075Disabling compacts and flushes for region at 1731412350075Disabling writes for close at 1731412350076 (+1 ms)Writing region close event to WAL at 1731412350076Closed at 1731412350076 2024-11-12T11:52:30,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741845_1021 (size=45) 2024-11-12T11:52:30,077 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 88db77d5150c2af8a9c27d5475a321fd, NAME => 'test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741845_1021 (size=45) 2024-11-12T11:52:30,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741845_1021 (size=45) 2024-11-12T11:52:30,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f3aaa68bd9553e5b9366c15cdfd85491, disabling compactions & flushes 2024-11-12T11:52:30,078 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:30,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:30,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. after waiting 0 ms 2024-11-12T11:52:30,079 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:30,079 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:30,079 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f3aaa68bd9553e5b9366c15cdfd85491: Waiting for close lock at 1731412350078Disabling compacts and flushes for region at 1731412350078Disabling writes for close at 1731412350078Writing region close event to WAL at 1731412350079 (+1 ms)Closed at 1731412350079 2024-11-12T11:52:30,079 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f828bb7c8aa989003de21e14a812e974, NAME => 'test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T11:52:30,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741846_1022 (size=45) 2024-11-12T11:52:30,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741846_1022 (size=45) 2024-11-12T11:52:30,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741846_1022 (size=45) 2024-11-12T11:52:30,086 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,086 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 9ae3a40ca66d508699f89dac1ef676d7, disabling compactions & flushes 2024-11-12T11:52:30,086 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7. 2024-11-12T11:52:30,086 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7. 2024-11-12T11:52:30,086 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7. after waiting 0 ms 2024-11-12T11:52:30,086 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7. 2024-11-12T11:52:30,086 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7. 2024-11-12T11:52:30,087 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9ae3a40ca66d508699f89dac1ef676d7: Waiting for close lock at 1731412350086Disabling compacts and flushes for region at 1731412350086Disabling writes for close at 1731412350086Writing region close event to WAL at 1731412350086Closed at 1731412350086 2024-11-12T11:52:30,087 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3a9bcab83bd9e8b9783e7f72eb485a37, NAME => 'test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741847_1023 (size=45) 2024-11-12T11:52:30,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741847_1023 (size=45) 2024-11-12T11:52:30,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741847_1023 (size=45) 2024-11-12T11:52:30,095 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,095 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 88db77d5150c2af8a9c27d5475a321fd, disabling compactions & flushes 2024-11-12T11:52:30,095 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:30,095 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:30,095 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. after waiting 0 ms 2024-11-12T11:52:30,095 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:30,095 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:30,095 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 88db77d5150c2af8a9c27d5475a321fd: Waiting for close lock at 1731412350095Disabling compacts and flushes for region at 1731412350095Disabling writes for close at 1731412350095Writing region close event to WAL at 1731412350095Closed at 1731412350095 2024-11-12T11:52:30,096 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 413d5888010bc8c2a41616f921b4b45e, NAME => 'test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741848_1024 (size=45) 2024-11-12T11:52:30,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741848_1024 (size=45) 2024-11-12T11:52:30,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741848_1024 (size=45) 2024-11-12T11:52:30,100 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f828bb7c8aa989003de21e14a812e974, disabling compactions & flushes 2024-11-12T11:52:30,101 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:30,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:30,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. after waiting 0 ms 2024-11-12T11:52:30,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:30,101 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:30,101 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f828bb7c8aa989003de21e14a812e974: Waiting for close lock at 1731412350100Disabling compacts and flushes for region at 1731412350100Disabling writes for close at 1731412350101 (+1 ms)Writing region close event to WAL at 1731412350101Closed at 1731412350101 2024-11-12T11:52:30,102 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 44845a612b40b8a8d7da0646602761b7, NAME => 'test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741849_1025 (size=45) 2024-11-12T11:52:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741850_1026 (size=45) 2024-11-12T11:52:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741849_1025 (size=45) 2024-11-12T11:52:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741850_1026 (size=45) 2024-11-12T11:52:30,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741849_1025 (size=45) 2024-11-12T11:52:30,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741850_1026 (size=45) 2024-11-12T11:52:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741851_1027 (size=45) 2024-11-12T11:52:30,119 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,119 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741851_1027 (size=45) 2024-11-12T11:52:30,119 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 413d5888010bc8c2a41616f921b4b45e, disabling compactions & flushes 2024-11-12T11:52:30,119 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3a9bcab83bd9e8b9783e7f72eb485a37, disabling compactions & flushes 2024-11-12T11:52:30,119 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:30,119 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:30,119 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:30,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741851_1027 (size=45) 2024-11-12T11:52:30,119 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:30,120 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. after waiting 0 ms 2024-11-12T11:52:30,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. after waiting 0 ms 2024-11-12T11:52:30,120 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:30,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:30,120 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:30,120 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:30,120 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 413d5888010bc8c2a41616f921b4b45e: Waiting for close lock at 1731412350119Disabling compacts and flushes for region at 1731412350119Disabling writes for close at 1731412350120 (+1 ms)Writing region close event to WAL at 1731412350120Closed at 1731412350120 2024-11-12T11:52:30,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3a9bcab83bd9e8b9783e7f72eb485a37: Waiting for close lock at 1731412350119Disabling compacts and flushes for region at 1731412350119Disabling writes for close at 1731412350120 (+1 ms)Writing region close event to WAL at 1731412350120Closed at 1731412350120 2024-11-12T11:52:30,121 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2d17593aab0898d03d839f7ddc825798, NAME => 'test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,121 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 914332156a93810ffdd86211f75b1742, NAME => 'test,ppp,1731412349920.914332156a93810ffdd86211f75b1742.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,121 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,121 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 44845a612b40b8a8d7da0646602761b7, disabling compactions & flushes 2024-11-12T11:52:30,122 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:30,122 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:30,122 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. after waiting 0 ms 2024-11-12T11:52:30,122 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:30,122 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:30,122 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 44845a612b40b8a8d7da0646602761b7: Waiting for close lock at 1731412350121Disabling compacts and flushes for region at 1731412350121Disabling writes for close at 1731412350122 (+1 ms)Writing region close event to WAL at 1731412350122Closed at 1731412350122 2024-11-12T11:52:30,123 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5a4df46f8ab1b85b3e12c8376d94d3d3, NAME => 'test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741852_1028 (size=45) 2024-11-12T11:52:30,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741852_1028 (size=45) 2024-11-12T11:52:30,134 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1731412349920.914332156a93810ffdd86211f75b1742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 914332156a93810ffdd86211f75b1742, disabling compactions & flushes 2024-11-12T11:52:30,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741852_1028 (size=45) 2024-11-12T11:52:30,135 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1731412349920.914332156a93810ffdd86211f75b1742. 2024-11-12T11:52:30,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731412349920.914332156a93810ffdd86211f75b1742. 2024-11-12T11:52:30,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731412349920.914332156a93810ffdd86211f75b1742. after waiting 0 ms 2024-11-12T11:52:30,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731412349920.914332156a93810ffdd86211f75b1742. 2024-11-12T11:52:30,135 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1731412349920.914332156a93810ffdd86211f75b1742. 2024-11-12T11:52:30,135 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 914332156a93810ffdd86211f75b1742: Waiting for close lock at 1731412350135Disabling compacts and flushes for region at 1731412350135Disabling writes for close at 1731412350135Writing region close event to WAL at 1731412350135Closed at 1731412350135 2024-11-12T11:52:30,136 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 569bd93fc79997920641cb7028f160e0, NAME => 'test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741853_1029 (size=45) 2024-11-12T11:52:30,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741853_1029 (size=45) 2024-11-12T11:52:30,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741853_1029 (size=45) 2024-11-12T11:52:30,139 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,140 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 2d17593aab0898d03d839f7ddc825798, disabling compactions & flushes 2024-11-12T11:52:30,140 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:30,140 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:30,140 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. after waiting 0 ms 2024-11-12T11:52:30,140 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:30,140 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:30,140 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2d17593aab0898d03d839f7ddc825798: Waiting for close lock at 1731412350140Disabling compacts and flushes for region at 1731412350140Disabling writes for close at 1731412350140Writing region close event to WAL at 1731412350140Closed at 1731412350140 2024-11-12T11:52:30,141 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0383ab7f8f8136da6c91952511d8d2e6, NAME => 'test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741854_1030 (size=45) 2024-11-12T11:52:30,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741854_1030 (size=45) 2024-11-12T11:52:30,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741854_1030 (size=45) 2024-11-12T11:52:30,143 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5a4df46f8ab1b85b3e12c8376d94d3d3, disabling compactions & flushes 2024-11-12T11:52:30,144 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:30,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:30,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. after waiting 0 ms 2024-11-12T11:52:30,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:30,144 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:30,144 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5a4df46f8ab1b85b3e12c8376d94d3d3: Waiting for close lock at 1731412350144Disabling compacts and flushes for region at 1731412350144Disabling writes for close at 1731412350144Writing region close event to WAL at 1731412350144Closed at 1731412350144 2024-11-12T11:52:30,145 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6a247bf888e4a967cccc4510e131fa0f, NAME => 'test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741855_1031 (size=45) 2024-11-12T11:52:30,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741855_1031 (size=45) 2024-11-12T11:52:30,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741855_1031 (size=45) 2024-11-12T11:52:30,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 569bd93fc79997920641cb7028f160e0, disabling compactions & flushes 2024-11-12T11:52:30,152 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:30,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:30,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. after waiting 0 ms 2024-11-12T11:52:30,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:30,152 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:30,152 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 569bd93fc79997920641cb7028f160e0: Waiting for close lock at 1731412350152Disabling compacts and flushes for region at 1731412350152Disabling writes for close at 1731412350152Writing region close event to WAL at 1731412350152Closed at 1731412350152 2024-11-12T11:52:30,153 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c34b19c841273fa15336639ee4fe2af6, NAME => 'test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741856_1032 (size=45) 2024-11-12T11:52:30,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741856_1032 (size=45) 2024-11-12T11:52:30,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741856_1032 (size=45) 2024-11-12T11:52:30,158 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0383ab7f8f8136da6c91952511d8d2e6, disabling compactions & flushes 2024-11-12T11:52:30,159 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:30,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:30,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. after waiting 0 ms 2024-11-12T11:52:30,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:30,159 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:30,159 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0383ab7f8f8136da6c91952511d8d2e6: Waiting for close lock at 1731412350159Disabling compacts and flushes for region at 1731412350159Disabling writes for close at 1731412350159Writing region close event to WAL at 1731412350159Closed at 1731412350159 2024-11-12T11:52:30,160 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c47427c2053fa4af37c7b70c21c77ef2, NAME => 'test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741857_1033 (size=45) 2024-11-12T11:52:30,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741857_1033 (size=45) 2024-11-12T11:52:30,163 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,163 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6a247bf888e4a967cccc4510e131fa0f, disabling compactions & flushes 2024-11-12T11:52:30,163 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:30,163 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:30,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. after waiting 1 ms 2024-11-12T11:52:30,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:30,164 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:30,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6a247bf888e4a967cccc4510e131fa0f: Waiting for close lock at 1731412350163Disabling compacts and flushes for region at 1731412350163Disabling writes for close at 1731412350164 (+1 ms)Writing region close event to WAL at 1731412350164Closed at 1731412350164 2024-11-12T11:52:30,165 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => efcf1d7a717f7ea5226c8f259d2cdbe8, NAME => 'test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741857_1033 (size=45) 2024-11-12T11:52:30,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741858_1034 (size=45) 2024-11-12T11:52:30,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741858_1034 (size=45) 2024-11-12T11:52:30,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741858_1034 (size=45) 2024-11-12T11:52:30,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c34b19c841273fa15336639ee4fe2af6, disabling compactions & flushes 2024-11-12T11:52:30,175 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. 2024-11-12T11:52:30,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. 2024-11-12T11:52:30,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. after waiting 0 ms 2024-11-12T11:52:30,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. 2024-11-12T11:52:30,175 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. 2024-11-12T11:52:30,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c34b19c841273fa15336639ee4fe2af6: Waiting for close lock at 1731412350175Disabling compacts and flushes for region at 1731412350175Disabling writes for close at 1731412350175Writing region close event to WAL at 1731412350175Closed at 1731412350175 2024-11-12T11:52:30,176 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c874e55257c5357c1082439ee6eda71a, NAME => 'test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741859_1035 (size=45) 2024-11-12T11:52:30,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741859_1035 (size=45) 2024-11-12T11:52:30,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741859_1035 (size=45) 2024-11-12T11:52:30,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,182 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c47427c2053fa4af37c7b70c21c77ef2, disabling compactions & flushes 2024-11-12T11:52:30,182 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:30,183 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:30,183 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. after waiting 0 ms 2024-11-12T11:52:30,183 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:30,183 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:30,183 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c47427c2053fa4af37c7b70c21c77ef2: Waiting for close lock at 1731412350182Disabling compacts and flushes for region at 1731412350182Disabling writes for close at 1731412350183 (+1 ms)Writing region close event to WAL at 1731412350183Closed at 1731412350183 2024-11-12T11:52:30,184 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f30fe248b58f81ca59f81dde9d522901, NAME => 'test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741860_1036 (size=45) 2024-11-12T11:52:30,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741860_1036 (size=45) 2024-11-12T11:52:30,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741860_1036 (size=45) 2024-11-12T11:52:30,189 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,190 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing efcf1d7a717f7ea5226c8f259d2cdbe8, disabling compactions & flushes 2024-11-12T11:52:30,190 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:30,190 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:30,190 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. after waiting 0 ms 2024-11-12T11:52:30,190 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:30,190 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:30,190 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for efcf1d7a717f7ea5226c8f259d2cdbe8: Waiting for close lock at 1731412350189Disabling compacts and flushes for region at 1731412350189Disabling writes for close at 1731412350190 (+1 ms)Writing region close event to WAL at 1731412350190Closed at 1731412350190 2024-11-12T11:52:30,191 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b33fbfeb60fca5b204bdf3a8333158e8, NAME => 'test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09 2024-11-12T11:52:30,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741861_1037 (size=45) 2024-11-12T11:52:30,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741861_1037 (size=45) 2024-11-12T11:52:30,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741861_1037 (size=45) 2024-11-12T11:52:30,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c874e55257c5357c1082439ee6eda71a, disabling compactions & flushes 2024-11-12T11:52:30,196 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:30,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:30,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. after waiting 0 ms 2024-11-12T11:52:30,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:30,196 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:30,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c874e55257c5357c1082439ee6eda71a: Waiting for close lock at 1731412350196Disabling compacts and flushes for region at 1731412350196Disabling writes for close at 1731412350196Writing region close event to WAL at 1731412350196Closed at 1731412350196 2024-11-12T11:52:30,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741862_1038 (size=45) 2024-11-12T11:52:30,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741862_1038 (size=45) 2024-11-12T11:52:30,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741862_1038 (size=45) 2024-11-12T11:52:30,199 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,200 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f30fe248b58f81ca59f81dde9d522901, disabling compactions & flushes 2024-11-12T11:52:30,200 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:30,200 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:30,200 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. after waiting 0 ms 2024-11-12T11:52:30,200 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:30,200 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:30,200 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f30fe248b58f81ca59f81dde9d522901: Waiting for close lock at 1731412350200Disabling compacts and flushes for region at 1731412350200Disabling writes for close at 1731412350200Writing region close event to WAL at 1731412350200Closed at 1731412350200 2024-11-12T11:52:30,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741863_1039 (size=42) 2024-11-12T11:52:30,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741863_1039 (size=42) 2024-11-12T11:52:30,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741863_1039 (size=42) 2024-11-12T11:52:30,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b33fbfeb60fca5b204bdf3a8333158e8, disabling compactions & flushes 2024-11-12T11:52:30,207 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:30,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:30,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. after waiting 0 ms 2024-11-12T11:52:30,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:30,207 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:30,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b33fbfeb60fca5b204bdf3a8333158e8: Waiting for close lock at 1731412350207Disabling compacts and flushes for region at 1731412350207Disabling writes for close at 1731412350207Writing region close event to WAL at 1731412350207Closed at 1731412350207 2024-11-12T11:52:30,210 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T11:52:30,215 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,215 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731412349920.57a12d2944d6f25f894da49363450447.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,215 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,215 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,215 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,216 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731412349920.914332156a93810ffdd86211f75b1742.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,217 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,218 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731412350210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731412350210"}]},"ts":"1731412350210"} 2024-11-12T11:52:30,263 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-12T11:52:30,265 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T11:52:30,268 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731412350266"}]},"ts":"1731412350266"} 2024-11-12T11:52:30,273 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-12T11:52:30,274 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {b51f50ce4a7c=0} racks are {/default-rack=0} 2024-11-12T11:52:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T11:52:30,297 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T11:52:30,297 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T11:52:30,297 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T11:52:30,297 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T11:52:30,297 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T11:52:30,297 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T11:52:30,297 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T11:52:30,297 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T11:52:30,297 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T11:52:30,297 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T11:52:30,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cbfb2956e6def690689f5e33134c9dd4, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57a12d2944d6f25f894da49363450447, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9e98aed23b0727af5f513c5de61a616, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f786dd883997cd82efafbe7d3d173fc7, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cf142f73f167aa11416e3ae10a6bf795, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f3aaa68bd9553e5b9366c15cdfd85491, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88db77d5150c2af8a9c27d5475a321fd, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f828bb7c8aa989003de21e14a812e974, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=413d5888010bc8c2a41616f921b4b45e, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=44845a612b40b8a8d7da0646602761b7, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2d17593aab0898d03d839f7ddc825798, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5a4df46f8ab1b85b3e12c8376d94d3d3, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0383ab7f8f8136da6c91952511d8d2e6, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a247bf888e4a967cccc4510e131fa0f, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c47427c2053fa4af37c7b70c21c77ef2, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=efcf1d7a717f7ea5226c8f259d2cdbe8, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f30fe248b58f81ca59f81dde9d522901, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b33fbfeb60fca5b204bdf3a8333158e8, ASSIGN}] 2024-11-12T11:52:30,312 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b33fbfeb60fca5b204bdf3a8333158e8, ASSIGN 2024-11-12T11:52:30,313 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b33fbfeb60fca5b204bdf3a8333158e8, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,315 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f30fe248b58f81ca59f81dde9d522901, ASSIGN 2024-11-12T11:52:30,316 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f30fe248b58f81ca59f81dde9d522901, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,318 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:30,319 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,38813,1731412347557; forceNewPlan=false, retain=false 2024-11-12T11:52:30,321 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=efcf1d7a717f7ea5226c8f259d2cdbe8, ASSIGN 2024-11-12T11:52:30,322 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=efcf1d7a717f7ea5226c8f259d2cdbe8, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,324 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c47427c2053fa4af37c7b70c21c77ef2, ASSIGN 2024-11-12T11:52:30,325 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c47427c2053fa4af37c7b70c21c77ef2, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,327 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:30,328 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,38813,1731412347557; forceNewPlan=false, retain=false 2024-11-12T11:52:30,329 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6a247bf888e4a967cccc4510e131fa0f, ASSIGN 2024-11-12T11:52:30,331 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6a247bf888e4a967cccc4510e131fa0f, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,332 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0383ab7f8f8136da6c91952511d8d2e6, ASSIGN 2024-11-12T11:52:30,334 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0383ab7f8f8136da6c91952511d8d2e6, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,335 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:30,336 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,38813,1731412347557; forceNewPlan=false, retain=false 2024-11-12T11:52:30,338 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5a4df46f8ab1b85b3e12c8376d94d3d3, ASSIGN 2024-11-12T11:52:30,339 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5a4df46f8ab1b85b3e12c8376d94d3d3, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,340 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:30,342 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,38813,1731412347557; forceNewPlan=false, retain=false 2024-11-12T11:52:30,343 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2d17593aab0898d03d839f7ddc825798, ASSIGN 2024-11-12T11:52:30,344 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2d17593aab0898d03d839f7ddc825798, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,346 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=44845a612b40b8a8d7da0646602761b7, ASSIGN 2024-11-12T11:52:30,347 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=44845a612b40b8a8d7da0646602761b7, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,348 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=413d5888010bc8c2a41616f921b4b45e, ASSIGN 2024-11-12T11:52:30,349 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=413d5888010bc8c2a41616f921b4b45e, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,351 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:30,352 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,38813,1731412347557; forceNewPlan=false, retain=false 2024-11-12T11:52:30,353 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f828bb7c8aa989003de21e14a812e974, ASSIGN 2024-11-12T11:52:30,354 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f828bb7c8aa989003de21e14a812e974, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,356 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88db77d5150c2af8a9c27d5475a321fd, ASSIGN 2024-11-12T11:52:30,357 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88db77d5150c2af8a9c27d5475a321fd, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,358 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:30,359 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,38813,1731412347557; forceNewPlan=false, retain=false 2024-11-12T11:52:30,360 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f3aaa68bd9553e5b9366c15cdfd85491, ASSIGN 2024-11-12T11:52:30,362 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f3aaa68bd9553e5b9366c15cdfd85491, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,363 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cf142f73f167aa11416e3ae10a6bf795, ASSIGN 2024-11-12T11:52:30,364 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cf142f73f167aa11416e3ae10a6bf795, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,366 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:30,368 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,38813,1731412347557; forceNewPlan=false, retain=false 2024-11-12T11:52:30,369 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f786dd883997cd82efafbe7d3d173fc7, ASSIGN 2024-11-12T11:52:30,371 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f786dd883997cd82efafbe7d3d173fc7, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,372 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9e98aed23b0727af5f513c5de61a616, ASSIGN 2024-11-12T11:52:30,373 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f9e98aed23b0727af5f513c5de61a616, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,375 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57a12d2944d6f25f894da49363450447, ASSIGN 2024-11-12T11:52:30,376 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=57a12d2944d6f25f894da49363450447, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,44417,1731412347465; forceNewPlan=false, retain=false 2024-11-12T11:52:30,377 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cbfb2956e6def690689f5e33134c9dd4, ASSIGN 2024-11-12T11:52:30,378 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cbfb2956e6def690689f5e33134c9dd4, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,34963,1731412347604; forceNewPlan=false, retain=false 2024-11-12T11:52:30,379 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:30,381 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN; state=OFFLINE, location=b51f50ce4a7c,38813,1731412347557; forceNewPlan=false, retain=false 2024-11-12T11:52:30,469 INFO [b51f50ce4a7c:37827 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-12T11:52:30,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,473 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=6a247bf888e4a967cccc4510e131fa0f, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,475 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=44845a612b40b8a8d7da0646602761b7, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b33fbfeb60fca5b204bdf3a8333158e8, ASSIGN because future has completed 2024-11-12T11:52:30,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f828bb7c8aa989003de21e14a812e974, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure b33fbfeb60fca5b204bdf3a8333158e8, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=efcf1d7a717f7ea5226c8f259d2cdbe8, ASSIGN because future has completed 2024-11-12T11:52:30,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure efcf1d7a717f7ea5226c8f259d2cdbe8, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6a247bf888e4a967cccc4510e131fa0f, ASSIGN because future has completed 2024-11-12T11:52:30,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5a4df46f8ab1b85b3e12c8376d94d3d3, ASSIGN because future has completed 2024-11-12T11:52:30,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=44845a612b40b8a8d7da0646602761b7, ASSIGN because future has completed 2024-11-12T11:52:30,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44845a612b40b8a8d7da0646602761b7, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f828bb7c8aa989003de21e14a812e974, ASSIGN because future has completed 2024-11-12T11:52:30,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f3aaa68bd9553e5b9366c15cdfd85491, ASSIGN because future has completed 2024-11-12T11:52:30,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3aaa68bd9553e5b9366c15cdfd85491, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f828bb7c8aa989003de21e14a812e974, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a4df46f8ab1b85b3e12c8376d94d3d3, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a247bf888e4a967cccc4510e131fa0f, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,516 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=f786dd883997cd82efafbe7d3d173fc7, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=57a12d2944d6f25f894da49363450447, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,518 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f786dd883997cd82efafbe7d3d173fc7, ASSIGN because future has completed 2024-11-12T11:52:30,519 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=569bd93fc79997920641cb7028f160e0, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure f786dd883997cd82efafbe7d3d173fc7, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=57a12d2944d6f25f894da49363450447, ASSIGN because future has completed 2024-11-12T11:52:30,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN because future has completed 2024-11-12T11:52:30,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN because future has completed 2024-11-12T11:52:30,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:30,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN because future has completed 2024-11-12T11:52:30,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:30,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:30,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a12d2944d6f25f894da49363450447, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:30,548 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=914332156a93810ffdd86211f75b1742, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,548 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,549 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,550 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,550 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f30fe248b58f81ca59f81dde9d522901, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN because future has completed 2024-11-12T11:52:30,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914332156a93810ffdd86211f75b1742, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:30,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN because future has completed 2024-11-12T11:52:30,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN because future has completed 2024-11-12T11:52:30,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN because future has completed 2024-11-12T11:52:30,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN because future has completed 2024-11-12T11:52:30,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:30,561 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f30fe248b58f81ca59f81dde9d522901, ASSIGN because future has completed 2024-11-12T11:52:30,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c47427c2053fa4af37c7b70c21c77ef2, ASSIGN because future has completed 2024-11-12T11:52:30,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c47427c2053fa4af37c7b70c21c77ef2, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f30fe248b58f81ca59f81dde9d522901, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:30,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:30,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,38813,1731412347557}] 2024-11-12T11:52:30,590 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,591 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=2d17593aab0898d03d839f7ddc825798, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,592 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=413d5888010bc8c2a41616f921b4b45e, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=88db77d5150c2af8a9c27d5475a321fd, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0383ab7f8f8136da6c91952511d8d2e6, ASSIGN because future has completed 2024-11-12T11:52:30,594 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=cf142f73f167aa11416e3ae10a6bf795, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0383ab7f8f8136da6c91952511d8d2e6, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2d17593aab0898d03d839f7ddc825798, ASSIGN because future has completed 2024-11-12T11:52:30,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=413d5888010bc8c2a41616f921b4b45e, ASSIGN because future has completed 2024-11-12T11:52:30,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88db77d5150c2af8a9c27d5475a321fd, ASSIGN because future has completed 2024-11-12T11:52:30,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88db77d5150c2af8a9c27d5475a321fd, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,601 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cf142f73f167aa11416e3ae10a6bf795, ASSIGN because future has completed 2024-11-12T11:52:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T11:52:30,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf142f73f167aa11416e3ae10a6bf795, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 413d5888010bc8c2a41616f921b4b45e, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d17593aab0898d03d839f7ddc825798, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f9e98aed23b0727af5f513c5de61a616, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=cbfb2956e6def690689f5e33134c9dd4, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f9e98aed23b0727af5f513c5de61a616, ASSIGN because future has completed 2024-11-12T11:52:30,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9e98aed23b0727af5f513c5de61a616, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cbfb2956e6def690689f5e33134c9dd4, ASSIGN because future has completed 2024-11-12T11:52:30,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbfb2956e6def690689f5e33134c9dd4, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:30,632 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T11:52:30,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45987, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T11:52:30,639 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:30,640 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => efcf1d7a717f7ea5226c8f259d2cdbe8, NAME => 'test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-12T11:52:30,640 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,640 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,640 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,641 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,643 INFO [StoreOpener-efcf1d7a717f7ea5226c8f259d2cdbe8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,645 INFO [StoreOpener-efcf1d7a717f7ea5226c8f259d2cdbe8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efcf1d7a717f7ea5226c8f259d2cdbe8 columnFamilyName family 2024-11-12T11:52:30,645 DEBUG [StoreOpener-efcf1d7a717f7ea5226c8f259d2cdbe8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,646 INFO [StoreOpener-efcf1d7a717f7ea5226c8f259d2cdbe8-1 {}] regionserver.HStore(327): Store=efcf1d7a717f7ea5226c8f259d2cdbe8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,646 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,647 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,647 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,648 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,648 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,650 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,653 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/efcf1d7a717f7ea5226c8f259d2cdbe8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,654 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened efcf1d7a717f7ea5226c8f259d2cdbe8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70420153, jitterRate=0.04934205114841461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,654 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:30,656 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for efcf1d7a717f7ea5226c8f259d2cdbe8: Running coprocessor pre-open hook at 1731412350641Writing region info on filesystem at 1731412350641Initializing all the Stores at 1731412350642 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350642Cleaning up temporary data from old regions at 1731412350648 (+6 ms)Running coprocessor post-open hooks at 1731412350654 (+6 ms)Region opened successfully at 1731412350656 (+2 ms) 2024-11-12T11:52:30,658 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8., pid=32, masterSystemTime=1731412350632 2024-11-12T11:52:30,661 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:30,661 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:30,661 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:30,662 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => b33fbfeb60fca5b204bdf3a8333158e8, NAME => 'test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-12T11:52:30,662 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,662 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,662 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,663 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,663 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,666 INFO [StoreOpener-b33fbfeb60fca5b204bdf3a8333158e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure efcf1d7a717f7ea5226c8f259d2cdbe8, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:30,668 INFO [StoreOpener-b33fbfeb60fca5b204bdf3a8333158e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b33fbfeb60fca5b204bdf3a8333158e8 columnFamilyName family 2024-11-12T11:52:30,668 DEBUG [StoreOpener-b33fbfeb60fca5b204bdf3a8333158e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,669 INFO [StoreOpener-b33fbfeb60fca5b204bdf3a8333158e8-1 {}] regionserver.HStore(327): Store=b33fbfeb60fca5b204bdf3a8333158e8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,669 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,671 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,671 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,672 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=27 2024-11-12T11:52:30,672 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure efcf1d7a717f7ea5226c8f259d2cdbe8, server=b51f50ce4a7c,44417,1731412347465 in 183 msec 2024-11-12T11:52:30,675 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=efcf1d7a717f7ea5226c8f259d2cdbe8, ASSIGN in 371 msec 2024-11-12T11:52:30,678 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/b33fbfeb60fca5b204bdf3a8333158e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,678 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened b33fbfeb60fca5b204bdf3a8333158e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75294066, jitterRate=0.12196901440620422}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,679 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:30,679 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for b33fbfeb60fca5b204bdf3a8333158e8: Running coprocessor pre-open hook at 1731412350663Writing region info on filesystem at 1731412350663Initializing all the Stores at 1731412350665 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350665Cleaning up temporary data from old regions at 1731412350672 (+7 ms)Running coprocessor post-open hooks at 1731412350679 (+7 ms)Region opened successfully at 1731412350679 2024-11-12T11:52:30,680 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8., pid=31, masterSystemTime=1731412350632 2024-11-12T11:52:30,685 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:30,685 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:30,685 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:30,685 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:30,685 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 569bd93fc79997920641cb7028f160e0, NAME => 'test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => f3aaa68bd9553e5b9366c15cdfd85491, NAME => 'test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,686 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,686 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,688 INFO [StoreOpener-569bd93fc79997920641cb7028f160e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,688 INFO [StoreOpener-f3aaa68bd9553e5b9366c15cdfd85491-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure b33fbfeb60fca5b204bdf3a8333158e8, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:30,691 INFO [StoreOpener-f3aaa68bd9553e5b9366c15cdfd85491-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3aaa68bd9553e5b9366c15cdfd85491 columnFamilyName family 2024-11-12T11:52:30,691 DEBUG [StoreOpener-f3aaa68bd9553e5b9366c15cdfd85491-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,691 INFO [StoreOpener-569bd93fc79997920641cb7028f160e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 569bd93fc79997920641cb7028f160e0 columnFamilyName family 2024-11-12T11:52:30,692 DEBUG [StoreOpener-569bd93fc79997920641cb7028f160e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,692 INFO [StoreOpener-f3aaa68bd9553e5b9366c15cdfd85491-1 {}] regionserver.HStore(327): Store=f3aaa68bd9553e5b9366c15cdfd85491/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,692 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,692 INFO [StoreOpener-569bd93fc79997920641cb7028f160e0-1 {}] regionserver.HStore(327): Store=569bd93fc79997920641cb7028f160e0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,692 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,693 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,694 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,694 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,694 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,694 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,694 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,695 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,695 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,697 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,697 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-12T11:52:30,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure b33fbfeb60fca5b204bdf3a8333158e8, server=b51f50ce4a7c,44417,1731412347465 in 215 msec 2024-11-12T11:52:30,700 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/569bd93fc79997920641cb7028f160e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,701 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f3aaa68bd9553e5b9366c15cdfd85491/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b33fbfeb60fca5b204bdf3a8333158e8, ASSIGN in 397 msec 2024-11-12T11:52:30,701 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 569bd93fc79997920641cb7028f160e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68077090, jitterRate=0.01442769169807434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,701 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:30,701 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 569bd93fc79997920641cb7028f160e0: Running coprocessor pre-open hook at 1731412350686Writing region info on filesystem at 1731412350686Initializing all the Stores at 1731412350688 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350688Cleaning up temporary data from old regions at 1731412350695 (+7 ms)Running coprocessor post-open hooks at 1731412350701 (+6 ms)Region opened successfully at 1731412350701 2024-11-12T11:52:30,702 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened f3aaa68bd9553e5b9366c15cdfd85491; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68293465, jitterRate=0.017651930451393127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,702 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:30,702 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for f3aaa68bd9553e5b9366c15cdfd85491: Running coprocessor pre-open hook at 1731412350686Writing region info on filesystem at 1731412350686Initializing all the Stores at 1731412350688 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350688Cleaning up temporary data from old regions at 1731412350695 (+7 ms)Running coprocessor post-open hooks at 1731412350702 (+7 ms)Region opened successfully at 1731412350702 2024-11-12T11:52:30,703 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0., pid=40, masterSystemTime=1731412350679 2024-11-12T11:52:30,703 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491., pid=34, masterSystemTime=1731412350632 2024-11-12T11:52:30,705 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:30,706 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:30,706 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,ppp,1731412349920.914332156a93810ffdd86211f75b1742. 2024-11-12T11:52:30,706 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 914332156a93810ffdd86211f75b1742, NAME => 'test,ppp,1731412349920.914332156a93810ffdd86211f75b1742.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-12T11:52:30,706 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=569bd93fc79997920641cb7028f160e0, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,707 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,ppp,1731412349920.914332156a93810ffdd86211f75b1742.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,707 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,707 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,707 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:30,707 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:30,707 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:30,707 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a247bf888e4a967cccc4510e131fa0f, NAME => 'test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-12T11:52:30,708 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,708 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,708 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,708 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,710 INFO [StoreOpener-914332156a93810ffdd86211f75b1742-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,38813,1731412347557 because future has completed 2024-11-12T11:52:30,711 INFO [StoreOpener-6a247bf888e4a967cccc4510e131fa0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,712 INFO [StoreOpener-914332156a93810ffdd86211f75b1742-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 914332156a93810ffdd86211f75b1742 columnFamilyName family 2024-11-12T11:52:30,712 DEBUG [StoreOpener-914332156a93810ffdd86211f75b1742-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,713 INFO [StoreOpener-6a247bf888e4a967cccc4510e131fa0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a247bf888e4a967cccc4510e131fa0f columnFamilyName family 2024-11-12T11:52:30,713 DEBUG [StoreOpener-6a247bf888e4a967cccc4510e131fa0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,713 INFO [StoreOpener-914332156a93810ffdd86211f75b1742-1 {}] regionserver.HStore(327): Store=914332156a93810ffdd86211f75b1742/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,713 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,714 INFO [StoreOpener-6a247bf888e4a967cccc4510e131fa0f-1 {}] regionserver.HStore(327): Store=6a247bf888e4a967cccc4510e131fa0f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,714 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,715 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,715 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3aaa68bd9553e5b9366c15cdfd85491, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:30,716 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,716 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,717 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,717 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,717 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,717 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,719 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T11:52:30,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51671, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T11:52:30,720 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,721 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=22 2024-11-12T11:52:30,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,38813,1731412347557 in 185 msec 2024-11-12T11:52:30,727 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:30,728 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/914332156a93810ffdd86211f75b1742/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,728 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 0383ab7f8f8136da6c91952511d8d2e6, NAME => 'test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-12T11:52:30,729 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,729 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 914332156a93810ffdd86211f75b1742; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74914720, jitterRate=0.11631631851196289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,729 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,729 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/6a247bf888e4a967cccc4510e131fa0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,729 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 914332156a93810ffdd86211f75b1742 2024-11-12T11:52:30,729 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,729 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 914332156a93810ffdd86211f75b1742: Running coprocessor pre-open hook at 1731412350707Writing region info on filesystem at 1731412350707Initializing all the Stores at 1731412350709 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350709Cleaning up temporary data from old regions at 1731412350717 (+8 ms)Running coprocessor post-open hooks at 1731412350729 (+12 ms)Region opened successfully at 1731412350729 2024-11-12T11:52:30,729 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN in 425 msec 2024-11-12T11:52:30,730 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 6a247bf888e4a967cccc4510e131fa0f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62202808, jitterRate=-0.07310593128204346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,731 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:30,731 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 6a247bf888e4a967cccc4510e131fa0f: Running coprocessor pre-open hook at 1731412350708Writing region info on filesystem at 1731412350708Initializing all the Stores at 1731412350710 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350710Cleaning up temporary data from old regions at 1731412350717 (+7 ms)Running coprocessor post-open hooks at 1731412350731 (+14 ms)Region opened successfully at 1731412350731 2024-11-12T11:52:30,731 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731412349920.914332156a93810ffdd86211f75b1742., pid=43, masterSystemTime=1731412350679 2024-11-12T11:52:30,733 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f., pid=37, masterSystemTime=1731412350632 2024-11-12T11:52:30,734 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731412349920.914332156a93810ffdd86211f75b1742. 2024-11-12T11:52:30,734 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,ppp,1731412349920.914332156a93810ffdd86211f75b1742. 2024-11-12T11:52:30,735 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. 2024-11-12T11:52:30,735 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 57a392190264a7b69a7d25f871aed2c7, NAME => 'test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-12T11:52:30,735 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,735 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,736 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,736 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,736 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:30,736 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:30,736 INFO [StoreOpener-0383ab7f8f8136da6c91952511d8d2e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,737 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:30,737 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 44845a612b40b8a8d7da0646602761b7, NAME => 'test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-12T11:52:30,737 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=12 2024-11-12T11:52:30,737 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure f3aaa68bd9553e5b9366c15cdfd85491, server=b51f50ce4a7c,44417,1731412347465 in 234 msec 2024-11-12T11:52:30,738 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,738 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,739 INFO [StoreOpener-57a392190264a7b69a7d25f871aed2c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,740 INFO [StoreOpener-0383ab7f8f8136da6c91952511d8d2e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0383ab7f8f8136da6c91952511d8d2e6 columnFamilyName family 2024-11-12T11:52:30,740 DEBUG [StoreOpener-0383ab7f8f8136da6c91952511d8d2e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,740 INFO [StoreOpener-44845a612b40b8a8d7da0646602761b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,741 INFO [StoreOpener-0383ab7f8f8136da6c91952511d8d2e6-1 {}] regionserver.HStore(327): Store=0383ab7f8f8136da6c91952511d8d2e6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,741 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f3aaa68bd9553e5b9366c15cdfd85491, ASSIGN in 437 msec 2024-11-12T11:52:30,742 INFO [StoreOpener-57a392190264a7b69a7d25f871aed2c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57a392190264a7b69a7d25f871aed2c7 columnFamilyName family 2024-11-12T11:52:30,742 DEBUG [StoreOpener-57a392190264a7b69a7d25f871aed2c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,743 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,744 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,745 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,745 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,745 INFO [StoreOpener-57a392190264a7b69a7d25f871aed2c7-1 {}] regionserver.HStore(327): Store=57a392190264a7b69a7d25f871aed2c7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,745 INFO [StoreOpener-44845a612b40b8a8d7da0646602761b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 44845a612b40b8a8d7da0646602761b7 columnFamilyName family 2024-11-12T11:52:30,745 DEBUG [StoreOpener-44845a612b40b8a8d7da0646602761b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,745 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,746 INFO [StoreOpener-44845a612b40b8a8d7da0646602761b7-1 {}] regionserver.HStore(327): Store=44845a612b40b8a8d7da0646602761b7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,747 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,748 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a247bf888e4a967cccc4510e131fa0f, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:30,748 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,749 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,749 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,749 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,750 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,751 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,751 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,751 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,751 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,755 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/0383ab7f8f8136da6c91952511d8d2e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,755 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a392190264a7b69a7d25f871aed2c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,756 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,756 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 0383ab7f8f8136da6c91952511d8d2e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71453098, jitterRate=0.06473413109779358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=24 2024-11-12T11:52:30,756 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:30,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a247bf888e4a967cccc4510e131fa0f, server=b51f50ce4a7c,44417,1731412347465 in 240 msec 2024-11-12T11:52:30,756 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 57a392190264a7b69a7d25f871aed2c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65344612, jitterRate=-0.026289403438568115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,756 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:30,757 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 0383ab7f8f8136da6c91952511d8d2e6: Running coprocessor pre-open hook at 1731412350730Writing region info on filesystem at 1731412350730Initializing all the Stores at 1731412350734 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350734Cleaning up temporary data from old regions at 1731412350745 (+11 ms)Running coprocessor post-open hooks at 1731412350756 (+11 ms)Region opened successfully at 1731412350756 2024-11-12T11:52:30,757 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 57a392190264a7b69a7d25f871aed2c7: Running coprocessor pre-open hook at 1731412350736Writing region info on filesystem at 1731412350736Initializing all the Stores at 1731412350738 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350738Cleaning up temporary data from old regions at 1731412350749 (+11 ms)Running coprocessor post-open hooks at 1731412350757 (+8 ms)Region opened successfully at 1731412350757 2024-11-12T11:52:30,758 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6., pid=50, masterSystemTime=1731412350718 2024-11-12T11:52:30,758 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7., pid=47, masterSystemTime=1731412350679 2024-11-12T11:52:30,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6a247bf888e4a967cccc4510e131fa0f, ASSIGN in 456 msec 2024-11-12T11:52:30,761 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. 2024-11-12T11:52:30,761 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/44845a612b40b8a8d7da0646602761b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,761 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. 2024-11-12T11:52:30,761 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:30,761 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a9bcab83bd9e8b9783e7f72eb485a37, NAME => 'test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-12T11:52:30,762 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,762 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,762 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,762 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 44845a612b40b8a8d7da0646602761b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70956312, jitterRate=0.05733144283294678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,762 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,762 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:30,763 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 44845a612b40b8a8d7da0646602761b7: Running coprocessor pre-open hook at 1731412350738Writing region info on filesystem at 1731412350738Initializing all the Stores at 1731412350739 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350739Cleaning up temporary data from old regions at 1731412350751 (+12 ms)Running coprocessor post-open hooks at 1731412350762 (+11 ms)Region opened successfully at 1731412350762 2024-11-12T11:52:30,763 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=57a392190264a7b69a7d25f871aed2c7, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,764 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:30,764 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:30,764 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:30,764 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 88db77d5150c2af8a9c27d5475a321fd, NAME => 'test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-12T11:52:30,765 INFO [StoreOpener-3a9bcab83bd9e8b9783e7f72eb485a37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,765 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,765 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,765 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,765 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,766 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7., pid=33, masterSystemTime=1731412350632 2024-11-12T11:52:30,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,38813,1731412347557 because future has completed 2024-11-12T11:52:30,768 INFO [StoreOpener-3a9bcab83bd9e8b9783e7f72eb485a37-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a9bcab83bd9e8b9783e7f72eb485a37 columnFamilyName family 2024-11-12T11:52:30,768 DEBUG [StoreOpener-3a9bcab83bd9e8b9783e7f72eb485a37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,769 INFO [StoreOpener-88db77d5150c2af8a9c27d5475a321fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,769 INFO [StoreOpener-3a9bcab83bd9e8b9783e7f72eb485a37-1 {}] regionserver.HStore(327): Store=3a9bcab83bd9e8b9783e7f72eb485a37/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,769 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,769 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,770 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:30,770 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:30,770 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:30,770 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,770 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 57a12d2944d6f25f894da49363450447, NAME => 'test,ccc,1731412349920.57a12d2944d6f25f894da49363450447.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-12T11:52:30,770 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,771 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ccc,1731412349920.57a12d2944d6f25f894da49363450447.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,771 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,771 INFO [StoreOpener-88db77d5150c2af8a9c27d5475a321fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88db77d5150c2af8a9c27d5475a321fd columnFamilyName family 2024-11-12T11:52:30,771 DEBUG [StoreOpener-88db77d5150c2af8a9c27d5475a321fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,772 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,772 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,772 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,772 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,773 INFO [StoreOpener-88db77d5150c2af8a9c27d5475a321fd-1 {}] regionserver.HStore(327): Store=88db77d5150c2af8a9c27d5475a321fd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,774 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0383ab7f8f8136da6c91952511d8d2e6, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:30,775 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,776 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,776 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,777 INFO [StoreOpener-57a12d2944d6f25f894da49363450447-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 44845a612b40b8a8d7da0646602761b7, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:30,778 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,778 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,781 INFO [StoreOpener-57a12d2944d6f25f894da49363450447-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57a12d2944d6f25f894da49363450447 columnFamilyName family 2024-11-12T11:52:30,781 DEBUG [StoreOpener-57a12d2944d6f25f894da49363450447-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,781 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/3a9bcab83bd9e8b9783e7f72eb485a37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,782 INFO [StoreOpener-57a12d2944d6f25f894da49363450447-1 {}] regionserver.HStore(327): Store=57a12d2944d6f25f894da49363450447/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,782 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 3a9bcab83bd9e8b9783e7f72eb485a37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69926741, jitterRate=0.04198963940143585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,782 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,782 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,783 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:30,783 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 3a9bcab83bd9e8b9783e7f72eb485a37: Running coprocessor pre-open hook at 1731412350763Writing region info on filesystem at 1731412350763Initializing all the Stores at 1731412350764 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350764Cleaning up temporary data from old regions at 1731412350772 (+8 ms)Running coprocessor post-open hooks at 1731412350783 (+11 ms)Region opened successfully at 1731412350783 2024-11-12T11:52:30,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=23 2024-11-12T11:52:30,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 0383ab7f8f8136da6c91952511d8d2e6, server=b51f50ce4a7c,34963,1731412347604 in 183 msec 2024-11-12T11:52:30,785 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37., pid=49, masterSystemTime=1731412350679 2024-11-12T11:52:30,787 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0383ab7f8f8136da6c91952511d8d2e6, ASSIGN in 482 msec 2024-11-12T11:52:30,788 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,788 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:30,788 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:30,788 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. 2024-11-12T11:52:30,789 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => c34b19c841273fa15336639ee4fe2af6, NAME => 'test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-12T11:52:30,789 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,789 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,789 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,789 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,790 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,791 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/88db77d5150c2af8a9c27d5475a321fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,792 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=test, region=44845a612b40b8a8d7da0646602761b7. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:30,792 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,792 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,792 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 88db77d5150c2af8a9c27d5475a321fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63245020, jitterRate=-0.057575762271881104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,792 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:30,793 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 88db77d5150c2af8a9c27d5475a321fd: Running coprocessor pre-open hook at 1731412350765Writing region info on filesystem at 1731412350765Initializing all the Stores at 1731412350768 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350768Cleaning up temporary data from old regions at 1731412350779 (+11 ms)Running coprocessor post-open hooks at 1731412350792 (+13 ms)Region opened successfully at 1731412350793 (+1 ms) 2024-11-12T11:52:30,794 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd., pid=51, masterSystemTime=1731412350718 2024-11-12T11:52:30,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,38813,1731412347557 because future has completed 2024-11-12T11:52:30,797 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,798 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:30,798 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:30,798 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:30,798 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => cf142f73f167aa11416e3ae10a6bf795, NAME => 'test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-12T11:52:30,798 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,799 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,799 INFO [StoreOpener-c34b19c841273fa15336639ee4fe2af6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,799 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,799 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,799 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,38813,1731412347557, table=test, region=57a392190264a7b69a7d25f871aed2c7. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:30,802 INFO [StoreOpener-c34b19c841273fa15336639ee4fe2af6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c34b19c841273fa15336639ee4fe2af6 columnFamilyName family 2024-11-12T11:52:30,802 DEBUG [StoreOpener-c34b19c841273fa15336639ee4fe2af6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,803 INFO [StoreOpener-c34b19c841273fa15336639ee4fe2af6-1 {}] regionserver.HStore(327): Store=c34b19c841273fa15336639ee4fe2af6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,803 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=16 2024-11-12T11:52:30,804 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,38813,1731412347557 in 212 msec 2024-11-12T11:52:30,805 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,806 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,806 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,807 INFO [StoreOpener-cf142f73f167aa11416e3ae10a6bf795-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN in 504 msec 2024-11-12T11:52:30,809 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,810 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,810 INFO [StoreOpener-cf142f73f167aa11416e3ae10a6bf795-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf142f73f167aa11416e3ae10a6bf795 columnFamilyName family 2024-11-12T11:52:30,810 DEBUG [StoreOpener-cf142f73f167aa11416e3ae10a6bf795-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,811 INFO [StoreOpener-cf142f73f167aa11416e3ae10a6bf795-1 {}] regionserver.HStore(327): Store=cf142f73f167aa11416e3ae10a6bf795/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,811 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,813 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,813 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,814 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,814 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88db77d5150c2af8a9c27d5475a321fd, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:30,818 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,820 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a12d2944d6f25f894da49363450447/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=18 2024-11-12T11:52:30,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 44845a612b40b8a8d7da0646602761b7, server=b51f50ce4a7c,44417,1731412347465 in 323 msec 2024-11-12T11:52:30,821 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 57a12d2944d6f25f894da49363450447; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68481360, jitterRate=0.020451784133911133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,821 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:30,821 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 57a12d2944d6f25f894da49363450447: Running coprocessor pre-open hook at 1731412350772Writing region info on filesystem at 1731412350772Initializing all the Stores at 1731412350776 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350776Cleaning up temporary data from old regions at 1731412350792 (+16 ms)Running coprocessor post-open hooks at 1731412350821 (+29 ms)Region opened successfully at 1731412350821 2024-11-12T11:52:30,822 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731412349920.57a12d2944d6f25f894da49363450447., pid=42, masterSystemTime=1731412350632 2024-11-12T11:52:30,824 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/cf142f73f167aa11416e3ae10a6bf795/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=44845a612b40b8a8d7da0646602761b7, ASSIGN in 519 msec 2024-11-12T11:52:30,824 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c34b19c841273fa15336639ee4fe2af6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,825 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened cf142f73f167aa11416e3ae10a6bf795; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67129719, jitterRate=3.1076371669769287E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,825 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:30,826 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for cf142f73f167aa11416e3ae10a6bf795: Running coprocessor pre-open hook at 1731412350799Writing region info on filesystem at 1731412350799Initializing all the Stores at 1731412350801 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350801Cleaning up temporary data from old regions at 1731412350814 (+13 ms)Running coprocessor post-open hooks at 1731412350825 (+11 ms)Region opened successfully at 1731412350825 2024-11-12T11:52:30,826 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:30,826 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:30,826 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened c34b19c841273fa15336639ee4fe2af6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60156039, jitterRate=-0.10360516607761383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,826 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:30,826 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for c34b19c841273fa15336639ee4fe2af6: Running coprocessor pre-open hook at 1731412350789Writing region info on filesystem at 1731412350789Initializing all the Stores at 1731412350792 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350792Cleaning up temporary data from old regions at 1731412350806 (+14 ms)Running coprocessor post-open hooks at 1731412350826 (+20 ms)Region opened successfully at 1731412350826 2024-11-12T11:52:30,827 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:30,827 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a4df46f8ab1b85b3e12c8376d94d3d3, NAME => 'test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-12T11:52:30,827 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795., pid=52, masterSystemTime=1731412350718 2024-11-12T11:52:30,828 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6., pid=39, masterSystemTime=1731412350679 2024-11-12T11:52:30,828 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,828 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,828 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,828 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=14 2024-11-12T11:52:30,832 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:30,832 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:30,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 88db77d5150c2af8a9c27d5475a321fd, server=b51f50ce4a7c,34963,1731412347604 in 225 msec 2024-11-12T11:52:30,832 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:30,832 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 413d5888010bc8c2a41616f921b4b45e, NAME => 'test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-12T11:52:30,833 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,834 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,834 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. 2024-11-12T11:52:30,834 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,834 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. 2024-11-12T11:52:30,834 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,834 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7. 2024-11-12T11:52:30,834 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 9ae3a40ca66d508699f89dac1ef676d7, NAME => 'test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-12T11:52:30,835 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,835 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,835 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,835 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=88db77d5150c2af8a9c27d5475a321fd, ASSIGN in 531 msec 2024-11-12T11:52:30,839 INFO [StoreOpener-5a4df46f8ab1b85b3e12c8376d94d3d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,839 INFO [StoreOpener-413d5888010bc8c2a41616f921b4b45e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,839 INFO [StoreOpener-9ae3a40ca66d508699f89dac1ef676d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,841 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c34b19c841273fa15336639ee4fe2af6, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,841 INFO [StoreOpener-413d5888010bc8c2a41616f921b4b45e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 413d5888010bc8c2a41616f921b4b45e columnFamilyName family 2024-11-12T11:52:30,841 INFO [StoreOpener-9ae3a40ca66d508699f89dac1ef676d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ae3a40ca66d508699f89dac1ef676d7 columnFamilyName family 2024-11-12T11:52:30,841 INFO [StoreOpener-5a4df46f8ab1b85b3e12c8376d94d3d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a4df46f8ab1b85b3e12c8376d94d3d3 columnFamilyName family 2024-11-12T11:52:30,841 DEBUG [StoreOpener-9ae3a40ca66d508699f89dac1ef676d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,841 DEBUG [StoreOpener-413d5888010bc8c2a41616f921b4b45e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,841 DEBUG [StoreOpener-5a4df46f8ab1b85b3e12c8376d94d3d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,842 INFO [StoreOpener-5a4df46f8ab1b85b3e12c8376d94d3d3-1 {}] regionserver.HStore(327): Store=5a4df46f8ab1b85b3e12c8376d94d3d3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,843 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,843 INFO [StoreOpener-9ae3a40ca66d508699f89dac1ef676d7-1 {}] regionserver.HStore(327): Store=9ae3a40ca66d508699f89dac1ef676d7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,843 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,844 INFO [StoreOpener-413d5888010bc8c2a41616f921b4b45e-1 {}] regionserver.HStore(327): Store=413d5888010bc8c2a41616f921b4b45e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,845 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,845 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,845 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,845 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,846 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,846 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,846 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,846 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,38813,1731412347557 because future has completed 2024-11-12T11:52:30,847 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,847 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,847 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,848 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,849 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,849 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,850 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf142f73f167aa11416e3ae10a6bf795, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:30,851 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,853 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,854 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/5a4df46f8ab1b85b3e12c8376d94d3d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,855 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 5a4df46f8ab1b85b3e12c8376d94d3d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74277863, jitterRate=0.10682640969753265}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,855 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:30,856 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 5a4df46f8ab1b85b3e12c8376d94d3d3: Running coprocessor pre-open hook at 1731412350829Writing region info on filesystem at 1731412350829Initializing all the Stores at 1731412350830 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350831 (+1 ms)Cleaning up temporary data from old regions at 1731412350846 (+15 ms)Running coprocessor post-open hooks at 1731412350855 (+9 ms)Region opened successfully at 1731412350855 2024-11-12T11:52:30,857 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/9ae3a40ca66d508699f89dac1ef676d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,857 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3., pid=36, masterSystemTime=1731412350632 2024-11-12T11:52:30,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=25 2024-11-12T11:52:30,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,38813,1731412347557 in 326 msec 2024-11-12T11:52:30,858 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 9ae3a40ca66d508699f89dac1ef676d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65792584, jitterRate=-0.019614100456237793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,858 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9ae3a40ca66d508699f89dac1ef676d7 2024-11-12T11:52:30,858 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 9ae3a40ca66d508699f89dac1ef676d7: Running coprocessor pre-open hook at 1731412350835Writing region info on filesystem at 1731412350835Initializing all the Stores at 1731412350837 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350837Cleaning up temporary data from old regions at 1731412350847 (+10 ms)Running coprocessor post-open hooks at 1731412350858 (+11 ms)Region opened successfully at 1731412350858 2024-11-12T11:52:30,858 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/413d5888010bc8c2a41616f921b4b45e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,860 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7., pid=48, masterSystemTime=1731412350679 2024-11-12T11:52:30,860 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 413d5888010bc8c2a41616f921b4b45e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68189026, jitterRate=0.016095668077468872}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,860 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:30,860 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 413d5888010bc8c2a41616f921b4b45e: Running coprocessor pre-open hook at 1731412350834Writing region info on filesystem at 1731412350834Initializing all the Stores at 1731412350836 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350836Cleaning up temporary data from old regions at 1731412350850 (+14 ms)Running coprocessor post-open hooks at 1731412350860 (+10 ms)Region opened successfully at 1731412350860 2024-11-12T11:52:30,861 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:30,861 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:30,861 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:30,861 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => f786dd883997cd82efafbe7d3d173fc7, NAME => 'test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-12T11:52:30,861 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e., pid=53, masterSystemTime=1731412350718 2024-11-12T11:52:30,862 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,862 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,862 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,862 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,863 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7. 2024-11-12T11:52:30,863 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,iii,1731412349920.9ae3a40ca66d508699f89dac1ef676d7. 2024-11-12T11:52:30,864 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0. 2024-11-12T11:52:30,864 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => a0b1117abc263d25216a06f7640a6dd0, NAME => 'test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-12T11:52:30,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN in 557 msec 2024-11-12T11:52:30,864 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,865 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,865 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,865 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,866 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:30,866 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:30,866 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:30,866 INFO [StoreOpener-f786dd883997cd82efafbe7d3d173fc7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,866 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => f30fe248b58f81ca59f81dde9d522901, NAME => 'test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-12T11:52:30,866 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,867 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,867 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,867 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,867 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,867 INFO [StoreOpener-a0b1117abc263d25216a06f7640a6dd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,869 INFO [StoreOpener-f786dd883997cd82efafbe7d3d173fc7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f786dd883997cd82efafbe7d3d173fc7 columnFamilyName family 2024-11-12T11:52:30,869 DEBUG [StoreOpener-f786dd883997cd82efafbe7d3d173fc7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,870 INFO [StoreOpener-f30fe248b58f81ca59f81dde9d522901-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,870 INFO [StoreOpener-f786dd883997cd82efafbe7d3d173fc7-1 {}] regionserver.HStore(327): Store=f786dd883997cd82efafbe7d3d173fc7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,870 INFO [StoreOpener-a0b1117abc263d25216a06f7640a6dd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0b1117abc263d25216a06f7640a6dd0 columnFamilyName family 2024-11-12T11:52:30,870 DEBUG [StoreOpener-a0b1117abc263d25216a06f7640a6dd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,870 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,38813,1731412347557 because future has completed 2024-11-12T11:52:30,871 INFO [StoreOpener-a0b1117abc263d25216a06f7640a6dd0-1 {}] regionserver.HStore(327): Store=a0b1117abc263d25216a06f7640a6dd0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,872 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,872 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,872 INFO [StoreOpener-f30fe248b58f81ca59f81dde9d522901-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f30fe248b58f81ca59f81dde9d522901 columnFamilyName family 2024-11-12T11:52:30,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,872 DEBUG [StoreOpener-f30fe248b58f81ca59f81dde9d522901-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,872 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,873 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,873 INFO [StoreOpener-f30fe248b58f81ca59f81dde9d522901-1 {}] regionserver.HStore(327): Store=f30fe248b58f81ca59f81dde9d522901/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,873 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,873 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,873 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,873 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,874 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,874 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,875 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,876 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 413d5888010bc8c2a41616f921b4b45e, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:30,877 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,877 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,877 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,877 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,881 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,881 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/a0b1117abc263d25216a06f7640a6dd0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,882 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened a0b1117abc263d25216a06f7640a6dd0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63534954, jitterRate=-0.05325540900230408}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,882 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0b1117abc263d25216a06f7640a6dd0 2024-11-12T11:52:30,882 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for a0b1117abc263d25216a06f7640a6dd0: Running coprocessor pre-open hook at 1731412350865Writing region info on filesystem at 1731412350865Initializing all the Stores at 1731412350867 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350867Cleaning up temporary data from old regions at 1731412350874 (+7 ms)Running coprocessor post-open hooks at 1731412350882 (+8 ms)Region opened successfully at 1731412350882 2024-11-12T11:52:30,883 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0., pid=44, masterSystemTime=1731412350679 2024-11-12T11:52:30,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=13 2024-11-12T11:52:30,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,38813,1731412347557 in 296 msec 2024-11-12T11:52:30,886 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0. 2024-11-12T11:52:30,886 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,,1731412349920.a0b1117abc263d25216a06f7640a6dd0. 2024-11-12T11:52:30,887 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:30,887 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => c874e55257c5357c1082439ee6eda71a, NAME => 'test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-12T11:52:30,888 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,888 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f786dd883997cd82efafbe7d3d173fc7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,888 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,888 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,888 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a0b1117abc263d25216a06f7640a6dd0, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,889 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,34963,1731412347604, table=test, region=cf142f73f167aa11416e3ae10a6bf795. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:30,889 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,34963,1731412347604, table=test, region=413d5888010bc8c2a41616f921b4b45e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:30,889 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened f786dd883997cd82efafbe7d3d173fc7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62964331, jitterRate=-0.06175835430622101}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,889 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:30,889 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for f786dd883997cd82efafbe7d3d173fc7: Running coprocessor pre-open hook at 1731412350862Writing region info on filesystem at 1731412350862Initializing all the Stores at 1731412350864 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350864Cleaning up temporary data from old regions at 1731412350873 (+9 ms)Running coprocessor post-open hooks at 1731412350889 (+16 ms)Region opened successfully at 1731412350889 2024-11-12T11:52:30,892 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7., pid=38, masterSystemTime=1731412350632 2024-11-12T11:52:30,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,38813,1731412347557 because future has completed 2024-11-12T11:52:30,895 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f30fe248b58f81ca59f81dde9d522901/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,896 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened f30fe248b58f81ca59f81dde9d522901; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73884549, jitterRate=0.10096557438373566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,896 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:30,896 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for f30fe248b58f81ca59f81dde9d522901: Running coprocessor pre-open hook at 1731412350867Writing region info on filesystem at 1731412350867Initializing all the Stores at 1731412350868 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350869 (+1 ms)Cleaning up temporary data from old regions at 1731412350877 (+8 ms)Running coprocessor post-open hooks at 1731412350896 (+19 ms)Region opened successfully at 1731412350896 2024-11-12T11:52:30,896 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:30,896 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:30,896 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:30,897 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => f828bb7c8aa989003de21e14a812e974, NAME => 'test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-12T11:52:30,897 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901., pid=46, masterSystemTime=1731412350718 2024-11-12T11:52:30,897 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,897 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,897 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,897 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,899 INFO [StoreOpener-c874e55257c5357c1082439ee6eda71a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN in 591 msec 2024-11-12T11:52:30,901 INFO [StoreOpener-f828bb7c8aa989003de21e14a812e974-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,902 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:30,903 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:30,903 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:30,903 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d17593aab0898d03d839f7ddc825798, NAME => 'test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-12T11:52:30,904 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,904 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,904 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,904 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,904 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,38813,1731412347557, table=test, region=a0b1117abc263d25216a06f7640a6dd0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:30,904 INFO [StoreOpener-f828bb7c8aa989003de21e14a812e974-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f828bb7c8aa989003de21e14a812e974 columnFamilyName family 2024-11-12T11:52:30,904 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,38813,1731412347557, table=test, region=57a392190264a7b69a7d25f871aed2c7. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:30,904 DEBUG [StoreOpener-f828bb7c8aa989003de21e14a812e974-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,905 INFO [StoreOpener-f828bb7c8aa989003de21e14a812e974-1 {}] regionserver.HStore(327): Store=f828bb7c8aa989003de21e14a812e974/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,906 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,906 INFO [StoreOpener-2d17593aab0898d03d839f7ddc825798-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,907 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,907 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,907 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,907 INFO [StoreOpener-c874e55257c5357c1082439ee6eda71a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c874e55257c5357c1082439ee6eda71a columnFamilyName family 2024-11-12T11:52:30,908 DEBUG [StoreOpener-c874e55257c5357c1082439ee6eda71a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,908 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,908 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,908 INFO [StoreOpener-c874e55257c5357c1082439ee6eda71a-1 {}] regionserver.HStore(327): Store=c874e55257c5357c1082439ee6eda71a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,908 INFO [StoreOpener-2d17593aab0898d03d839f7ddc825798-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d17593aab0898d03d839f7ddc825798 columnFamilyName family 2024-11-12T11:52:30,909 DEBUG [StoreOpener-2d17593aab0898d03d839f7ddc825798-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,909 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,910 INFO [StoreOpener-2d17593aab0898d03d839f7ddc825798-1 {}] regionserver.HStore(327): Store=2d17593aab0898d03d839f7ddc825798/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,910 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,910 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure f30fe248b58f81ca59f81dde9d522901, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:30,912 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,912 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,912 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,913 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,913 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,913 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,914 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,914 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,914 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,916 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure f786dd883997cd82efafbe7d3d173fc7, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:30,917 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,919 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f828bb7c8aa989003de21e14a812e974/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,921 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened f828bb7c8aa989003de21e14a812e974; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68502890, jitterRate=0.020772606134414673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,921 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:30,921 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for f828bb7c8aa989003de21e14a812e974: Running coprocessor pre-open hook at 1731412350898Writing region info on filesystem at 1731412350898Initializing all the Stores at 1731412350901 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350901Cleaning up temporary data from old regions at 1731412350908 (+7 ms)Running coprocessor post-open hooks at 1731412350921 (+13 ms)Region opened successfully at 1731412350921 2024-11-12T11:52:30,922 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974., pid=35, masterSystemTime=1731412350632 2024-11-12T11:52:30,923 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/2d17593aab0898d03d839f7ddc825798/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,925 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c874e55257c5357c1082439ee6eda71a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,925 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 2d17593aab0898d03d839f7ddc825798; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59785526, jitterRate=-0.10912624001502991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,925 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:30,926 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 2d17593aab0898d03d839f7ddc825798: Running coprocessor pre-open hook at 1731412350904Writing region info on filesystem at 1731412350904Initializing all the Stores at 1731412350906 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350906Cleaning up temporary data from old regions at 1731412350914 (+8 ms)Running coprocessor post-open hooks at 1731412350925 (+11 ms)Region opened successfully at 1731412350925 2024-11-12T11:52:30,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=9 2024-11-12T11:52:30,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure f786dd883997cd82efafbe7d3d173fc7, server=b51f50ce4a7c,44417,1731412347465 in 400 msec 2024-11-12T11:52:30,926 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened c874e55257c5357c1082439ee6eda71a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66559690, jitterRate=-0.008183330297470093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,926 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:30,926 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:30,926 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for c874e55257c5357c1082439ee6eda71a: Running coprocessor pre-open hook at 1731412350888Writing region info on filesystem at 1731412350888Initializing all the Stores at 1731412350892 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350892Cleaning up temporary data from old regions at 1731412350913 (+21 ms)Running coprocessor post-open hooks at 1731412350926 (+13 ms)Region opened successfully at 1731412350926 2024-11-12T11:52:30,926 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:30,927 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a., pid=41, masterSystemTime=1731412350679 2024-11-12T11:52:30,927 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798., pid=54, masterSystemTime=1731412350718 2024-11-12T11:52:30,928 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:30,930 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:30,931 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:30,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f828bb7c8aa989003de21e14a812e974, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:30,932 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:30,933 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:30,933 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:30,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f786dd883997cd82efafbe7d3d173fc7, ASSIGN in 629 msec 2024-11-12T11:52:30,933 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => f9e98aed23b0727af5f513c5de61a616, NAME => 'test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-12T11:52:30,933 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,934 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,934 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,934 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d17593aab0898d03d839f7ddc825798, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:30,938 INFO [StoreOpener-f9e98aed23b0727af5f513c5de61a616-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,941 INFO [StoreOpener-f9e98aed23b0727af5f513c5de61a616-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9e98aed23b0727af5f513c5de61a616 columnFamilyName family 2024-11-12T11:52:30,941 DEBUG [StoreOpener-f9e98aed23b0727af5f513c5de61a616-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,942 INFO [StoreOpener-f9e98aed23b0727af5f513c5de61a616-1 {}] regionserver.HStore(327): Store=f9e98aed23b0727af5f513c5de61a616/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,942 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=15 2024-11-12T11:52:30,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure f828bb7c8aa989003de21e14a812e974, server=b51f50ce4a7c,44417,1731412347465 in 436 msec 2024-11-12T11:52:30,943 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,944 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,944 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,944 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f828bb7c8aa989003de21e14a812e974, ASSIGN in 641 msec 2024-11-12T11:52:30,947 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,950 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f9e98aed23b0727af5f513c5de61a616/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=19 2024-11-12T11:52:30,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d17593aab0898d03d839f7ddc825798, server=b51f50ce4a7c,34963,1731412347604 in 334 msec 2024-11-12T11:52:30,951 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened f9e98aed23b0727af5f513c5de61a616; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60672560, jitterRate=-0.09590840339660645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,951 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:30,952 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for f9e98aed23b0727af5f513c5de61a616: Running coprocessor pre-open hook at 1731412350934Writing region info on filesystem at 1731412350934Initializing all the Stores at 1731412350937 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350938 (+1 ms)Cleaning up temporary data from old regions at 1731412350944 (+6 ms)Running coprocessor post-open hooks at 1731412350951 (+7 ms)Region opened successfully at 1731412350951 2024-11-12T11:52:30,953 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616., pid=55, masterSystemTime=1731412350718 2024-11-12T11:52:30,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2d17593aab0898d03d839f7ddc825798, ASSIGN in 650 msec 2024-11-12T11:52:30,955 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=c874e55257c5357c1082439ee6eda71a, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:30,955 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:30,955 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:30,956 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:30,956 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => cbfb2956e6def690689f5e33134c9dd4, NAME => 'test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-12T11:52:30,956 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,956 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,956 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,957 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,958 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,38813,1731412347557 because future has completed 2024-11-12T11:52:30,959 INFO [StoreOpener-cbfb2956e6def690689f5e33134c9dd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,959 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,961 INFO [StoreOpener-cbfb2956e6def690689f5e33134c9dd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cbfb2956e6def690689f5e33134c9dd4 columnFamilyName family 2024-11-12T11:52:30,961 DEBUG [StoreOpener-cbfb2956e6def690689f5e33134c9dd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,962 INFO [StoreOpener-cbfb2956e6def690689f5e33134c9dd4-1 {}] regionserver.HStore(327): Store=cbfb2956e6def690689f5e33134c9dd4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,962 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9e98aed23b0727af5f513c5de61a616, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:30,963 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,964 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,964 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,964 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,967 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=28 2024-11-12T11:52:30,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,38813,1731412347557 in 425 msec 2024-11-12T11:52:30,970 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/cbfb2956e6def690689f5e33134c9dd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN in 666 msec 2024-11-12T11:52:30,970 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened cbfb2956e6def690689f5e33134c9dd4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71395691, jitterRate=0.06387870013713837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,971 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:30,971 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for cbfb2956e6def690689f5e33134c9dd4: Running coprocessor pre-open hook at 1731412350957Writing region info on filesystem at 1731412350957Initializing all the Stores at 1731412350958 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350958Cleaning up temporary data from old regions at 1731412350965 (+7 ms)Running coprocessor post-open hooks at 1731412350971 (+6 ms)Region opened successfully at 1731412350971 2024-11-12T11:52:30,972 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4., pid=56, masterSystemTime=1731412350718 2024-11-12T11:52:30,975 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:30,975 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:30,975 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:30,975 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => c47427c2053fa4af37c7b70c21c77ef2, NAME => 'test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-12T11:52:30,976 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,976 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:30,976 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,976 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=8 2024-11-12T11:52:30,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure f9e98aed23b0727af5f513c5de61a616, server=b51f50ce4a7c,34963,1731412347604 in 353 msec 2024-11-12T11:52:30,978 INFO [StoreOpener-c47427c2053fa4af37c7b70c21c77ef2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f9e98aed23b0727af5f513c5de61a616, ASSIGN in 676 msec 2024-11-12T11:52:30,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:30,981 INFO [StoreOpener-c47427c2053fa4af37c7b70c21c77ef2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c47427c2053fa4af37c7b70c21c77ef2 columnFamilyName family 2024-11-12T11:52:30,981 DEBUG [StoreOpener-c47427c2053fa4af37c7b70c21c77ef2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:30,982 INFO [StoreOpener-c47427c2053fa4af37c7b70c21c77ef2-1 {}] regionserver.HStore(327): Store=c47427c2053fa4af37c7b70c21c77ef2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:30,982 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,983 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure cbfb2956e6def690689f5e33134c9dd4, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:30,984 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,985 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,985 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,988 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=29 2024-11-12T11:52:30,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure f30fe248b58f81ca59f81dde9d522901, server=b51f50ce4a7c,34963,1731412347604 in 415 msec 2024-11-12T11:52:30,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f30fe248b58f81ca59f81dde9d522901, ASSIGN in 687 msec 2024-11-12T11:52:30,991 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c47427c2053fa4af37c7b70c21c77ef2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T11:52:30,993 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened c47427c2053fa4af37c7b70c21c77ef2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63155271, jitterRate=-0.05891312658786774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:30,994 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:30,994 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for c47427c2053fa4af37c7b70c21c77ef2: Running coprocessor pre-open hook at 1731412350977Writing region info on filesystem at 1731412350977Initializing all the Stores at 1731412350978 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412350978Cleaning up temporary data from old regions at 1731412350985 (+7 ms)Running coprocessor post-open hooks at 1731412350994 (+9 ms)Region opened successfully at 1731412350994 2024-11-12T11:52:30,995 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,34963,1731412347604, table=test, region=cf142f73f167aa11416e3ae10a6bf795. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:30,995 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,34963,1731412347604, table=test, region=413d5888010bc8c2a41616f921b4b45e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:30,995 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2., pid=45, masterSystemTime=1731412350718 2024-11-12T11:52:30,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-12T11:52:30,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure cbfb2956e6def690689f5e33134c9dd4, server=b51f50ce4a7c,34963,1731412347604 in 368 msec 2024-11-12T11:52:30,999 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:30,999 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:30,999 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:31,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c47427c2053fa4af37c7b70c21c77ef2, server=b51f50ce4a7c,34963,1731412347604 because future has completed 2024-11-12T11:52:31,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cbfb2956e6def690689f5e33134c9dd4, ASSIGN in 700 msec 2024-11-12T11:52:31,009 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,38813,1731412347557, table=test, region=a0b1117abc263d25216a06f7640a6dd0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:31,009 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,38813,1731412347557, table=test, region=57a392190264a7b69a7d25f871aed2c7. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:31,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=26 2024-11-12T11:52:31,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure c47427c2053fa4af37c7b70c21c77ef2, server=b51f50ce4a7c,34963,1731412347604 in 440 msec 2024-11-12T11:52:31,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c47427c2053fa4af37c7b70c21c77ef2, ASSIGN in 709 msec 2024-11-12T11:52:31,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=5 2024-11-12T11:52:31,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,38813,1731412347557 in 454 msec 2024-11-12T11:52:31,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN in 716 msec 2024-11-12T11:52:31,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=17 2024-11-12T11:52:31,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 413d5888010bc8c2a41616f921b4b45e, server=b51f50ce4a7c,34963,1731412347604 in 410 msec 2024-11-12T11:52:31,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=413d5888010bc8c2a41616f921b4b45e, ASSIGN in 721 msec 2024-11-12T11:52:31,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:31,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5a4df46f8ab1b85b3e12c8376d94d3d3, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:31,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=11 2024-11-12T11:52:31,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure cf142f73f167aa11416e3ae10a6bf795, server=b51f50ce4a7c,34963,1731412347604 in 422 msec 2024-11-12T11:52:31,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cf142f73f167aa11416e3ae10a6bf795, ASSIGN in 730 msec 2024-11-12T11:52:31,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=21 2024-11-12T11:52:31,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 5a4df46f8ab1b85b3e12c8376d94d3d3, server=b51f50ce4a7c,44417,1731412347465 in 526 msec 2024-11-12T11:52:31,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5a4df46f8ab1b85b3e12c8376d94d3d3, ASSIGN in 735 msec 2024-11-12T11:52:31,039 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=57a12d2944d6f25f894da49363450447, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:31,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a12d2944d6f25f894da49363450447, server=b51f50ce4a7c,44417,1731412347465 because future has completed 2024-11-12T11:52:31,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=10 2024-11-12T11:52:31,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,38813,1731412347557 in 466 msec 2024-11-12T11:52:31,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN in 746 msec 2024-11-12T11:52:31,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=7 2024-11-12T11:52:31,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 57a12d2944d6f25f894da49363450447, server=b51f50ce4a7c,44417,1731412347465 in 506 msec 2024-11-12T11:52:31,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=57a12d2944d6f25f894da49363450447, ASSIGN in 751 msec 2024-11-12T11:52:31,054 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=914332156a93810ffdd86211f75b1742, regionState=OPEN, openSeqNum=2, regionLocation=b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:31,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914332156a93810ffdd86211f75b1742, server=b51f50ce4a7c,38813,1731412347557 because future has completed 2024-11-12T11:52:31,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=20 2024-11-12T11:52:31,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 914332156a93810ffdd86211f75b1742, server=b51f50ce4a7c,38813,1731412347557 in 506 msec 2024-11-12T11:52:31,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=4 2024-11-12T11:52:31,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN in 759 msec 2024-11-12T11:52:31,065 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T11:52:31,065 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731412351065"}]},"ts":"1731412351065"} 2024-11-12T11:52:31,068 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-12T11:52:31,070 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T11:52:31,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.1410 sec 2024-11-12T11:52:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T11:52:31,112 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-12T11:52:31,112 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-12T11:52:31,113 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T11:52:31,135 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-12T11:52:31,135 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T11:52:31,136 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-12T11:52:31,137 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T11:52:31,142 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-12T11:52:31,165 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-12T11:52:31,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T11:52:31,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T11:52:31,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T11:52:31,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T11:52:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cbfb2956e6def690689f5e33134c9dd4 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f9e98aed23b0727af5f513c5de61a616 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cf142f73f167aa11416e3ae10a6bf795 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 88db77d5150c2af8a9c27d5475a321fd changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 413d5888010bc8c2a41616f921b4b45e changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2d17593aab0898d03d839f7ddc825798 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0383ab7f8f8136da6c91952511d8d2e6 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c47427c2053fa4af37c7b70c21c77ef2 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f30fe248b58f81ca59f81dde9d522901 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a0b1117abc263d25216a06f7640a6dd0 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 57a392190264a7b69a7d25f871aed2c7 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9ae3a40ca66d508699f89dac1ef676d7 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3a9bcab83bd9e8b9783e7f72eb485a37 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 914332156a93810ffdd86211f75b1742 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 569bd93fc79997920641cb7028f160e0 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c34b19c841273fa15336639ee4fe2af6 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c874e55257c5357c1082439ee6eda71a changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 57a12d2944d6f25f894da49363450447 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f786dd883997cd82efafbe7d3d173fc7 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f3aaa68bd9553e5b9366c15cdfd85491 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f828bb7c8aa989003de21e14a812e974 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 44845a612b40b8a8d7da0646602761b7 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5a4df46f8ab1b85b3e12c8376d94d3d3 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6a247bf888e4a967cccc4510e131fa0f changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region efcf1d7a717f7ea5226c8f259d2cdbe8 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b33fbfeb60fca5b204bdf3a8333158e8 changed from -1.0 to 0.0, refreshing cache 2024-11-12T11:52:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-12T11:52:31,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-12T11:52:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-12T11:52:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(204): Hosts are {b51f50ce4a7c=0} racks are {/default-rack=0} 2024-11-12T11:52:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-12T11:52:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-12T11:52:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-12T11:52:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T11:52:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T11:52:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T11:52:31,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T11:52:31,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T11:52:31,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T11:52:31,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-12T11:52:31,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-12T11:52:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-12T11:52:31,262 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T11:52:31,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-12T11:52:31,291 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=381, OpenFileDescriptor=651, MaxFileDescriptor=1048576, SystemLoadAverage=119, ProcessCount=11, AvailableMemoryMB=1647 2024-11-12T11:52:31,294 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-12T11:52:31,295 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-12T11:52:31,295 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:31,296 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b51f50ce4a7c,38813,1731412347557: Simulated kill ***** 2024-11-12T11:52:31,296 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T11:52:31,298 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T11:52:31,301 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T11:52:31,302 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T11:52:31,302 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T11:52:31,302 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 292788936 }, "NonHeapMemoryUsage": { "committed": 150470656, "init": 7667712, "max": -1, "used": 148176992 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ] } 2024-11-12T11:52:31,305 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37827 {}] master.MasterRpcServices(700): b51f50ce4a7c,38813,1731412347557 reported a fatal error: ***** ABORTING region server b51f50ce4a7c,38813,1731412347557: Simulated kill ***** 2024-11-12T11:52:31,307 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b51f50ce4a7c,38813,1731412347557' ***** 2024-11-12T11:52:31,308 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-12T11:52:31,308 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T11:52:31,308 INFO [RS:1;b51f50ce4a7c:38813 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-12T11:52:31,308 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T11:52:31,308 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T11:52:31,308 INFO [RS:1;b51f50ce4a7c:38813 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-12T11:52:31,309 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T11:52:31,309 INFO [RS:1;b51f50ce4a7c:38813 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b51f50ce4a7c:38813. 2024-11-12T11:52:31,309 DEBUG [RS:1;b51f50ce4a7c:38813 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T11:52:31,315 DEBUG [RS:1;b51f50ce4a7c:38813 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:31,321 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta not finished, retry = 0 2024-11-12T11:52:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741836_1012 (size=32114) 2024-11-12T11:52:31,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741836_1012 (size=32114) 2024-11-12T11:52:31,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741836_1012 (size=32114) 2024-11-12T11:52:31,348 INFO [regionserver/b51f50ce4a7c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T11:52:31,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741834_1010 (size=2232) 2024-11-12T11:52:31,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741834_1010 (size=2232) 2024-11-12T11:52:31,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741834_1010 (size=2232) 2024-11-12T11:52:31,429 DEBUG [RS:1;b51f50ce4a7c:38813 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:31,429 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T11:52:31,430 INFO [RS:1;b51f50ce4a7c:38813 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38813 2024-11-12T11:52:31,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T11:52:31,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b51f50ce4a7c,38813,1731412347557 2024-11-12T11:52:31,480 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T11:52:31,493 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b51f50ce4a7c,38813,1731412347557] 2024-11-12T11:52:31,504 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b51f50ce4a7c,38813,1731412347557 already deleted, retry=false 2024-11-12T11:52:31,504 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b51f50ce4a7c,38813,1731412347557 on b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:31,508 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:31,510 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T11:52:31,512 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:31,512 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for b51f50ce4a7c,38813,1731412347557 (carryingMeta=true) b51f50ce4a7c,38813,1731412347557/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2ddc28b5[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T11:52:31,515 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true, isMeta: true 2024-11-12T11:52:31,517 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting 2024-11-12T11:52:31,518 INFO [PEWorker-1 {}] master.SplitWALManager(105): b51f50ce4a7c,38813,1731412347557 WAL count=1, meta=true 2024-11-12T11:52:31,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta}] 2024-11-12T11:52:31,526 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:31,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:31,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T11:52:31,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38813-0x101302d66c30002, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T11:52:31,593 INFO [RS:1;b51f50ce4a7c:38813 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T11:52:31,593 INFO [RS:1;b51f50ce4a7c:38813 {}] regionserver.HRegionServer(1031): Exiting; stopping=b51f50ce4a7c,38813,1731412347557; zookeeper connection closed. 2024-11-12T11:52:31,594 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@189e19be {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@189e19be 2024-11-12T11:52:31,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-12T11:52:31,716 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, size=31.4 K (32114bytes) 2024-11-12T11:52:31,717 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-12T11:52:31,717 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-12T11:52:31,717 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta 2024-11-12T11:52:31,723 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta after 4ms 2024-11-12T11:52:31,731 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta took 16ms 2024-11-12T11:52:31,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-12T11:52:31,746 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta.temp 2024-11-12T11:52:31,749 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta.temp 2024-11-12T11:52:31,752 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta so closing down 2024-11-12T11:52:31,753 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T11:52:31,754 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T11:52:31,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741864_1040 (size=31873) 2024-11-12T11:52:31,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741864_1040 (size=31873) 2024-11-12T11:52:31,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741864_1040 (size=31873) 2024-11-12T11:52:32,096 INFO [regionserver/b51f50ce4a7c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T11:52:32,096 INFO [regionserver/b51f50ce4a7c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T11:52:32,166 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta.temp (wrote 57 edits, skipped 0 edits in 4 ms) 2024-11-12T11:52:32,169 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta.temp to hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-12T11:52:32,174 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 436 ms; skipped=1; WAL=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-12T11:52:32,174 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, journal: Splitting hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, size=31.4 K (32114bytes) at 1731412351716Creating recovered edits writer path=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta.temp at 1731412351749 (+33 ms)Finishing writing output for hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta so closing down at 1731412351752 (+3 ms)3 split writer threads finished at 1731412351754 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta.temp (wrote 57 edits, skipped 0 edits in 4 ms) at 1731412352166 (+412 ms)Rename recovered edits hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta.temp to hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731412352169 (+3 ms)Processed 58 edits across 1 Regions in 436 ms; skipped=1; WAL=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731412352174 (+5 ms) 2024-11-12T11:52:32,175 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta 2024-11-12T11:52:32,176 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-12T11:52:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-12T11:52:32,186 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta to hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/oldWALs 2024-11-12T11:52:32,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T11:52:32,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 in 657 msec 2024-11-12T11:52:32,191 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:32,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-12T11:52:32,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 in 671 msec 2024-11-12T11:52:32,196 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting dir is empty, no logs to split. 2024-11-12T11:52:32,196 INFO [PEWorker-1 {}] master.SplitWALManager(105): b51f50ce4a7c,38813,1731412347557 WAL count=0, meta=true 2024-11-12T11:52:32,196 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if b51f50ce4a7c,38813,1731412347557 WAL splitting is done? wals=0, meta=true 2024-11-12T11:52:32,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T11:52:32,200 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:32,201 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:32,351 DEBUG [b51f50ce4a7c:37827 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T11:52:32,351 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(204): Hosts are {b51f50ce4a7c=0} racks are {/default-rack=0} 2024-11-12T11:52:32,352 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T11:52:32,352 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T11:52:32,352 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T11:52:32,352 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T11:52:32,352 INFO [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T11:52:32,352 INFO [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T11:52:32,352 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T11:52:32,352 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:32,354 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPENING 2024-11-12T11:52:32,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:32,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:32,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:32,397 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:32,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:32,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:32,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:32,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:32,556 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T11:52:32,556 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T11:52:32,556 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T11:52:32,558 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b51f50ce4a7c%2C44417%2C1731412347465.meta, suffix=.meta, logDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,44417,1731412347465, archiveDir=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/oldWALs, maxLogs=32 2024-11-12T11:52:32,572 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,44417,1731412347465/b51f50ce4a7c%2C44417%2C1731412347465.meta.1731412352560.meta, exclude list is [], retry=0 2024-11-12T11:52:32,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44441,DS-2f88c530-085d-4189-a3bb-e2e8f41f6ab8,DISK] 2024-11-12T11:52:32,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39411,DS-452caef6-73c2-4535-8909-6cc1964991cc,DISK] 2024-11-12T11:52:32,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42055,DS-8937733f-a602-4fa6-8252-e0ec7163a924,DISK] 2024-11-12T11:52:32,577 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,44417,1731412347465/b51f50ce4a7c%2C44417%2C1731412347465.meta.1731412352560.meta 2024-11-12T11:52:32,578 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42399:42399),(127.0.0.1/127.0.0.1:40641:40641),(127.0.0.1/127.0.0.1:33001:33001)] 2024-11-12T11:52:32,578 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T11:52:32,579 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T11:52:32,579 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T11:52:32,579 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T11:52:32,579 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T11:52:32,579 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:32,579 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T11:52:32,579 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T11:52:32,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T11:52:32,583 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T11:52:32,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:32,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:32,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T11:52:32,585 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T11:52:32,585 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:32,586 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:32,586 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T11:52:32,587 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T11:52:32,587 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:32,588 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:32,588 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T11:52:32,590 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T11:52:32,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:32,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T11:52:32,590 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T11:52:32,591 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740 2024-11-12T11:52:32,594 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740 2024-11-12T11:52:32,595 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-12T11:52:32,609 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-12T11:52:32,613 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-12T11:52:32,676 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/info/fefdd2a8ed954983b71f95826385c91a is 138, key is test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4./info:regioninfo/1731412350981/Put/seqid=0 2024-11-12T11:52:32,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741866_1042 (size=34924) 2024-11-12T11:52:32,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741866_1042 (size=34924) 2024-11-12T11:52:32,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741866_1042 (size=34924) 2024-11-12T11:52:32,688 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/info/fefdd2a8ed954983b71f95826385c91a 2024-11-12T11:52:32,747 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/ns/39c9b154c7784eafa0aaae27a3193c80 is 43, key is default/ns:d/1731412349710/Put/seqid=0 2024-11-12T11:52:32,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741867_1043 (size=5153) 2024-11-12T11:52:32,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741867_1043 (size=5153) 2024-11-12T11:52:32,758 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/ns/39c9b154c7784eafa0aaae27a3193c80 2024-11-12T11:52:32,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741867_1043 (size=5153) 2024-11-12T11:52:32,787 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/table/d383ca8eafc54f309df0f0f21601bc00 is 40, key is test/table:state/1731412351065/Put/seqid=0 2024-11-12T11:52:32,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741868_1044 (size=5165) 2024-11-12T11:52:32,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741868_1044 (size=5165) 2024-11-12T11:52:32,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741868_1044 (size=5165) 2024-11-12T11:52:32,797 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/table/d383ca8eafc54f309df0f0f21601bc00 2024-11-12T11:52:32,806 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/info/fefdd2a8ed954983b71f95826385c91a as hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/info/fefdd2a8ed954983b71f95826385c91a 2024-11-12T11:52:32,815 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/info/fefdd2a8ed954983b71f95826385c91a, entries=260, sequenceid=60, filesize=34.1 K 2024-11-12T11:52:32,818 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/ns/39c9b154c7784eafa0aaae27a3193c80 as hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/ns/39c9b154c7784eafa0aaae27a3193c80 2024-11-12T11:52:32,828 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/ns/39c9b154c7784eafa0aaae27a3193c80, entries=2, sequenceid=60, filesize=5.0 K 2024-11-12T11:52:32,830 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/table/d383ca8eafc54f309df0f0f21601bc00 as hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/table/d383ca8eafc54f309df0f0f21601bc00 2024-11-12T11:52:32,839 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/table/d383ca8eafc54f309df0f0f21601bc00, entries=2, sequenceid=60, filesize=5.0 K 2024-11-12T11:52:32,843 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 229ms, sequenceid=60, compaction requested=false; wal=null 2024-11-12T11:52:32,844 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T11:52:32,847 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-12T11:52:32,849 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T11:52:32,849 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T11:52:32,850 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T11:52:32,851 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T11:52:32,854 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-12T11:52:32,856 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61678175, jitterRate=-0.08092357218265533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T11:52:32,856 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T11:52:32,856 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731412352579Writing region info on filesystem at 1731412352579Initializing all the Stores at 1731412352581 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412352581Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412352581Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412352581Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731412352581Obtaining lock to block concurrent updates at 1731412352613 (+32 ms)Preparing flush snapshotting stores in 1588230740 at 1731412352613Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731412352615 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731412352616 (+1 ms)Flushing 1588230740/info: creating writer at 1731412352617 (+1 ms)Flushing 1588230740/info: appending metadata at 1731412352667 (+50 ms)Flushing 1588230740/info: closing flushed file at 1731412352669 (+2 ms)Flushing 1588230740/ns: creating writer at 1731412352732 (+63 ms)Flushing 1588230740/ns: appending metadata at 1731412352746 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731412352746Flushing 1588230740/table: creating writer at 1731412352767 (+21 ms)Flushing 1588230740/table: appending metadata at 1731412352786 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731412352786Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f30acd3: reopening flushed file at 1731412352805 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7071923b: reopening flushed file at 1731412352817 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75280aa1: reopening flushed file at 1731412352828 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 229ms, sequenceid=60, compaction requested=false; wal=null at 1731412352843 (+15 ms)Cleaning up temporary data from old regions at 1731412352849 (+6 ms)Running coprocessor post-open hooks at 1731412352856 (+7 ms)Region opened successfully at 1731412352856 2024-11-12T11:52:32,858 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731412352551 2024-11-12T11:52:32,861 DEBUG [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T11:52:32,861 INFO [RS_OPEN_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T11:52:32,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:32,865 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T11:52:32,865 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:32,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:32,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:32,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:32,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:32,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:32,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:32,921 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:32,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T11:52:32,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 in 524 msec 2024-11-12T11:52:32,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-12T11:52:32,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 727 msec 2024-11-12T11:52:32,929 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): b51f50ce4a7c,38813,1731412347557 had 9 regions 2024-11-12T11:52:32,930 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true, isMeta: false 2024-11-12T11:52:32,932 INFO [PEWorker-1 {}] master.SplitWALManager(105): b51f50ce4a7c,38813,1731412347557 WAL count=1, meta=false 2024-11-12T11:52:32,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187}] 2024-11-12T11:52:32,935 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:32,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:33,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34963 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-12T11:52:33,106 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, size=2.2 K (2232bytes) 2024-11-12T11:52:33,106 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187 2024-11-12T11:52:33,107 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187 after 1ms 2024-11-12T11:52:33,110 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187 took 4ms 2024-11-12T11:52:33,113 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 569bd93fc79997920641cb7028f160e0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T11:52:33,115 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 914332156a93810ffdd86211f75b1742: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T11:52:33,117 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 57a392190264a7b69a7d25f871aed2c7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T11:52:33,118 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 3a9bcab83bd9e8b9783e7f72eb485a37: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T11:52:33,120 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c34b19c841273fa15336639ee4fe2af6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T11:52:33,122 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 9ae3a40ca66d508699f89dac1ef676d7: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T11:52:33,124 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a0b1117abc263d25216a06f7640a6dd0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T11:52:33,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c874e55257c5357c1082439ee6eda71a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-12T11:52:33,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187 so closing down 2024-11-12T11:52:33,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T11:52:33,126 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T11:52:33,126 INFO [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 15 ms; skipped=8; WAL=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, size=2.2 K, length=2232, corrupted=false, cancelled=false 2024-11-12T11:52:33,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, journal: Splitting hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, size=2.2 K (2232bytes) at 1731412353106Finishing writing output for hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187 so closing down at 1731412353126 (+20 ms)3 split writer threads finished at 1731412353126Processed 8 edits across 0 Regions in 15 ms; skipped=8; WAL=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, size=2.2 K, length=2232, corrupted=false, cancelled=false at 1731412353126 2024-11-12T11:52:33,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187 2024-11-12T11:52:33,126 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b51f50ce4a7c:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-12T11:52:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37827 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-12T11:52:33,131 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting/b51f50ce4a7c%2C38813%2C1731412347557.1731412349187 to hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/oldWALs 2024-11-12T11:52:33,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T11:52:33,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 in 195 msec 2024-11-12T11:52:33,134 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-12T11:52:33,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 in 203 msec 2024-11-12T11:52:33,139 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,38813,1731412347557-splitting dir is empty, no logs to split. 2024-11-12T11:52:33,139 INFO [PEWorker-1 {}] master.SplitWALManager(105): b51f50ce4a7c,38813,1731412347557 WAL count=0, meta=false 2024-11-12T11:52:33,139 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if b51f50ce4a7c,38813,1731412347557 WAL splitting is done? wals=0, meta=false 2024-11-12T11:52:33,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN}] 2024-11-12T11:52:33,143 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,144 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:33,145 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:33,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,148 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:33,149 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,149 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:33,150 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,151 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:33,152 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,152 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:33,153 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,154 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:33,154 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,155 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T11:52:33,295 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(204): Hosts are {b51f50ce4a7c=0} racks are {/default-rack=0} 2024-11-12T11:52:33,295 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T11:52:33,295 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T11:52:33,295 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T11:52:33,295 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T11:52:33,295 INFO [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T11:52:33,295 INFO [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T11:52:33,295 DEBUG [b51f50ce4a7c:37827 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T11:52:33,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,297 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=914332156a93810ffdd86211f75b1742, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,297 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b51f50ce4a7c:38813 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T11:52:33,298 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, regionLocation=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=569bd93fc79997920641cb7028f160e0, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1, error=java.net.ConnectException: Call to address=b51f50ce4a7c:38813 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T11:52:33,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 from cache 2024-11-12T11:52:33,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, regionLocation=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,301 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T11:52:33,301 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,44417,1731412347465, seqNum=-1] 2024-11-12T11:52:33,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address b51f50ce4a7c:38813 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,301 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T11:52:33,302 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b51f50ce4a7c:38813 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T11:52:33,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,44417,1731412347465, seqNum=-1, error=java.net.ConnectException: Call to address=b51f50ce4a7c:38813 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,44417,1731412347465, seqNum=-1, error=java.net.ConnectException: Call to address=b51f50ce4a7c:38813 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,44417,1731412347465, seqNum=-1, error=java.net.ConnectException: Call to address=b51f50ce4a7c:38813 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,44417,1731412347465, seqNum=-1, error=java.net.ConnectException: Call to address=b51f50ce4a7c:38813 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49475, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T11:52:33,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,44417,1731412347465, seqNum=-1, error=java.net.ConnectException: Call to address=b51f50ce4a7c:38813 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,38813,1731412347557, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=b51f50ce4a7c,44417,1731412347465, seqNum=-1, error=java.net.ConnectException: Call to address=b51f50ce4a7c:38813 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address b51f50ce4a7c:38813 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: b51f50ce4a7c/172.17.0.2:38813 2024-11-12T11:52:33,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN because future has completed 2024-11-12T11:52:33,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:33,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN because future has completed 2024-11-12T11:52:33,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:33,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN because future has completed 2024-11-12T11:52:33,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN because future has completed 2024-11-12T11:52:33,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN because future has completed 2024-11-12T11:52:33,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:33,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN because future has completed 2024-11-12T11:52:33,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN because future has completed 2024-11-12T11:52:33,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN because future has completed 2024-11-12T11:52:33,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:33,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:33,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465}] 2024-11-12T11:52:33,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:33,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914332156a93810ffdd86211f75b1742, server=b51f50ce4a7c,34963,1731412347604}] 2024-11-12T11:52:33,463 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:33,463 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => c874e55257c5357c1082439ee6eda71a, NAME => 'test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-12T11:52:33,464 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,464 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:33,464 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,464 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,465 INFO [StoreOpener-c874e55257c5357c1082439ee6eda71a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,466 INFO [StoreOpener-c874e55257c5357c1082439ee6eda71a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c874e55257c5357c1082439ee6eda71a columnFamilyName family 2024-11-12T11:52:33,467 DEBUG [StoreOpener-c874e55257c5357c1082439ee6eda71a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:33,467 INFO [StoreOpener-c874e55257c5357c1082439ee6eda71a-1 {}] regionserver.HStore(327): Store=c874e55257c5357c1082439ee6eda71a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:33,467 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,468 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,470 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,470 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,470 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,472 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,473 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened c874e55257c5357c1082439ee6eda71a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62698995, jitterRate=-0.06571216881275177}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:33,473 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:33,473 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for c874e55257c5357c1082439ee6eda71a: Running coprocessor pre-open hook at 1731412353464Writing region info on filesystem at 1731412353464Initializing all the Stores at 1731412353465 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412353465Cleaning up temporary data from old regions at 1731412353470 (+5 ms)Running coprocessor post-open hooks at 1731412353473 (+3 ms)Region opened successfully at 1731412353473 2024-11-12T11:52:33,474 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a., pid=77, masterSystemTime=1731412353458 2024-11-12T11:52:33,476 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:33,476 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:33,477 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:33,477 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a9bcab83bd9e8b9783e7f72eb485a37, NAME => 'test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-12T11:52:33,477 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,477 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:33,477 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,477 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,479 INFO [StoreOpener-3a9bcab83bd9e8b9783e7f72eb485a37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,479 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-12T11:52:33,480 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,480 INFO [StoreOpener-3a9bcab83bd9e8b9783e7f72eb485a37-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a9bcab83bd9e8b9783e7f72eb485a37 columnFamilyName family 2024-11-12T11:52:33,480 DEBUG [StoreOpener-3a9bcab83bd9e8b9783e7f72eb485a37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:33,480 INFO [StoreOpener-3a9bcab83bd9e8b9783e7f72eb485a37-1 {}] regionserver.HStore(327): Store=3a9bcab83bd9e8b9783e7f72eb485a37/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:33,481 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,480 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,481 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,483 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,483 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,483 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,486 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,487 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 3a9bcab83bd9e8b9783e7f72eb485a37; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61596063, jitterRate=-0.08214713633060455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:33,487 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:33,487 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 3a9bcab83bd9e8b9783e7f72eb485a37: Running coprocessor pre-open hook at 1731412353477Writing region info on filesystem at 1731412353477Initializing all the Stores at 1731412353478 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412353479 (+1 ms)Cleaning up temporary data from old regions at 1731412353483 (+4 ms)Running coprocessor post-open hooks at 1731412353487 (+4 ms)Region opened successfully at 1731412353487 2024-11-12T11:52:33,488 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37., pid=75, masterSystemTime=1731412353458 2024-11-12T11:52:33,490 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:33,490 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:33,491 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:33,491 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 569bd93fc79997920641cb7028f160e0, NAME => 'test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-12T11:52:33,491 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,491 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:33,492 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,492 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,491 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-12T11:52:33,494 INFO [StoreOpener-569bd93fc79997920641cb7028f160e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,494 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,494 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914332156a93810ffdd86211f75b1742, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,495 INFO [StoreOpener-569bd93fc79997920641cb7028f160e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 569bd93fc79997920641cb7028f160e0 columnFamilyName family 2024-11-12T11:52:33,495 DEBUG [StoreOpener-569bd93fc79997920641cb7028f160e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:33,495 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914332156a93810ffdd86211f75b1742, server=b51f50ce4a7c,34963,1731412347604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,496 INFO [StoreOpener-569bd93fc79997920641cb7028f160e0-1 {}] regionserver.HStore(327): Store=569bd93fc79997920641cb7028f160e0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:33,496 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,497 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,498 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,499 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,499 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,501 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,502 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 569bd93fc79997920641cb7028f160e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58749664, jitterRate=-0.12456178665161133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:33,502 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:33,502 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 569bd93fc79997920641cb7028f160e0: Running coprocessor pre-open hook at 1731412353492Writing region info on filesystem at 1731412353492Initializing all the Stores at 1731412353493 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412353493Cleaning up temporary data from old regions at 1731412353499 (+6 ms)Running coprocessor post-open hooks at 1731412353502 (+3 ms)Region opened successfully at 1731412353502 2024-11-12T11:52:33,503 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0., pid=76, masterSystemTime=1731412353458 2024-11-12T11:52:33,505 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:33,506 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:33,506 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. 2024-11-12T11:52:33,506 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 57a392190264a7b69a7d25f871aed2c7, NAME => 'test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-12T11:52:33,506 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,506 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:33,506 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,506 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,509 INFO [StoreOpener-57a392190264a7b69a7d25f871aed2c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,510 INFO [StoreOpener-57a392190264a7b69a7d25f871aed2c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 57a392190264a7b69a7d25f871aed2c7 columnFamilyName family 2024-11-12T11:52:33,510 DEBUG [StoreOpener-57a392190264a7b69a7d25f871aed2c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:33,511 INFO [StoreOpener-57a392190264a7b69a7d25f871aed2c7-1 {}] regionserver.HStore(327): Store=57a392190264a7b69a7d25f871aed2c7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:33,511 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,513 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,515 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,515 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,515 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,515 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,516 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,516 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,516 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,517 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,517 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,517 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,517 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,518 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,520 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,522 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 57a392190264a7b69a7d25f871aed2c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67387049, jitterRate=0.00414527952671051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:33,522 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 57a392190264a7b69a7d25f871aed2c7 2024-11-12T11:52:33,522 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 57a392190264a7b69a7d25f871aed2c7: Running coprocessor pre-open hook at 1731412353507Writing region info on filesystem at 1731412353507Initializing all the Stores at 1731412353508 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412353508Cleaning up temporary data from old regions at 1731412353517 (+9 ms)Running coprocessor post-open hooks at 1731412353522 (+5 ms)Region opened successfully at 1731412353522 2024-11-12T11:52:33,523 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7., pid=72, masterSystemTime=1731412353458 2024-11-12T11:52:33,524 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,527 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,527 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "b51f50ce4a7c" port: 44417 start_code: 1731412347465 } transition { transition_code: OPENED region_info { region_id: 1731412349920 table_name { namespace: "default" qualifier: "test" } start_key: "fff" end_key: "ggg" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 72 initiating_master_active_time: 1731412347711 }; retry (#0) after 1003ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-12T11:52:33,528 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,528 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,528 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,528 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,528 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,528 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,528 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,528 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,531 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914332156a93810ffdd86211f75b1742, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,539 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,539 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914332156a93810ffdd86211f75b1742, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,539 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-11-12T11:52:33,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,541 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,542 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,570 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. 2024-11-12T11:52:33,570 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => c34b19c841273fa15336639ee4fe2af6, NAME => 'test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-12T11:52:33,571 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,571 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T11:52:33,571 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,571 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,572 INFO [StoreOpener-c34b19c841273fa15336639ee4fe2af6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,573 INFO [StoreOpener-c34b19c841273fa15336639ee4fe2af6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c34b19c841273fa15336639ee4fe2af6 columnFamilyName family 2024-11-12T11:52:33,573 DEBUG [StoreOpener-c34b19c841273fa15336639ee4fe2af6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T11:52:33,574 INFO [StoreOpener-c34b19c841273fa15336639ee4fe2af6-1 {}] regionserver.HStore(327): Store=c34b19c841273fa15336639ee4fe2af6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T11:52:33,574 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,575 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,577 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,577 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,577 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,579 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,581 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened c34b19c841273fa15336639ee4fe2af6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74479663, jitterRate=0.10983346402645111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T11:52:33,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,581 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c34b19c841273fa15336639ee4fe2af6 2024-11-12T11:52:33,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,581 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for c34b19c841273fa15336639ee4fe2af6: Running coprocessor pre-open hook at 1731412353571Writing region info on filesystem at 1731412353571Initializing all the Stores at 1731412353572 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731412353572Cleaning up temporary data from old regions at 1731412353577 (+5 ms)Running coprocessor post-open hooks at 1731412353581 (+4 ms)Region opened successfully at 1731412353581 2024-11-12T11:52:33,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,581 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,582 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,582 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6., pid=73, masterSystemTime=1731412353565 2024-11-12T11:52:33,582 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383582 2024-11-12T11:52:33,583 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,583 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,583 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,584 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,587 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.CallRunner(138): callId: 64 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 126 connection: 172.17.0.2:39843 deadline: 1731412363583, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-11-12T11:52:33,587 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,587 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,588 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,34963,1731412347604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,588 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "b51f50ce4a7c" port: 34963 start_code: 1731412347604 } transition { transition_code: OPENED region_info { region_id: 1731412349920 table_name { namespace: "default" qualifier: "test" } start_key: "uuu" end_key: "vvv" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 73 initiating_master_active_time: 1731412347711 }; retry (#0) after 1004ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-12T11:52:33,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,617 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,618 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,619 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,620 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,621 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,622 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,622 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413622 2024-11-12T11:52:33,622 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473622 2024-11-12T11:52:33,622 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 39 msec 2024-11-12T11:52:33,622 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,622 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,622 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,622 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,623 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,623 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,623 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,623 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,624 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,628 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,629 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,630 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,634 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,635 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,635 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9ae3a40ca66d508699f89dac1ef676d7, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,636 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-12T11:52:33,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,637 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,637 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,646 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,647 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,648 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383648 2024-11-12T11:52:33,648 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,648 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,649 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,651 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,652 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,652 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,660 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,661 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,662 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,662 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413662 2024-11-12T11:52:33,662 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473662 2024-11-12T11:52:33,662 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-12T11:52:33,663 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,663 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,663 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,663 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,663 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,663 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,674 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,674 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,675 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,675 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,675 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,675 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,675 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,675 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,675 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,675 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,675 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-12T11:52:33,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,676 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,677 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,684 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,684 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,684 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,684 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,685 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383685 2024-11-12T11:52:33,685 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,685 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,687 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,688 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,700 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,700 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413700 2024-11-12T11:52:33,700 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473700 2024-11-12T11:52:33,700 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-12T11:52:33,700 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,700 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,700 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,700 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,700 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,700 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,700 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,701 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,702 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,705 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,709 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,710 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,710 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,710 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,710 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,711 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,711 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,711 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,711 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,711 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,711 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 569bd93fc79997920641cb7028f160e0, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-12T11:52:33,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,712 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,712 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,718 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,718 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,722 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383723 2024-11-12T11:52:33,723 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,723 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,724 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,725 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,730 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,736 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,736 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413736 2024-11-12T11:52:33,736 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473736 2024-11-12T11:52:33,736 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-12T11:52:33,736 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,736 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,736 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,736 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,737 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,737 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,737 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,737 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,738 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,742 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,742 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,742 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,747 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,747 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,747 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,748 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,749 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,749 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,750 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a9bcab83bd9e8b9783e7f72eb485a37, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-12T11:52:33,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,751 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,751 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,759 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,759 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,759 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,760 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,760 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,760 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,760 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,760 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,760 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,760 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,760 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,761 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383761 2024-11-12T11:52:33,761 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,761 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,762 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,763 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,774 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413774 2024-11-12T11:52:33,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473774 2024-11-12T11:52:33,774 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-12T11:52:33,774 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,774 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,775 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,775 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,775 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,775 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,775 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,776 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,781 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,787 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,787 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0b1117abc263d25216a06f7640a6dd0, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-12T11:52:33,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,789 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,789 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,796 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,796 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,796 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,797 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,797 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383797 2024-11-12T11:52:33,798 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,798 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,799 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,800 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,809 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413809 2024-11-12T11:52:33,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473809 2024-11-12T11:52:33,809 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-12T11:52:33,809 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,810 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,810 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,810 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,810 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,810 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,811 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,821 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,821 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,821 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure c34b19c841273fa15336639ee4fe2af6, server=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,821 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-12T11:52:33,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,823 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,823 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,823 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,824 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,834 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,835 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,836 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383836 2024-11-12T11:52:33,836 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,836 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,837 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,838 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,838 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,847 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,848 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413847 2024-11-12T11:52:33,848 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473848 2024-11-12T11:52:33,848 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-12T11:52:33,848 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,848 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,848 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,848 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,848 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,849 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,849 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,850 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,854 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,859 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,859 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,860 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,860 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,860 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,860 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,860 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,860 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,860 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,860 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,860 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 57a392190264a7b69a7d25f871aed2c7, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,860 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-12T11:52:33,861 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,861 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,862 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,871 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,871 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,871 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,872 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,872 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,872 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,872 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,872 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,872 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,872 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,873 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,873 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383873 2024-11-12T11:52:33,873 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,873 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,874 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,875 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,876 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,876 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,883 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,883 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413883 2024-11-12T11:52:33,884 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473884 2024-11-12T11:52:33,884 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-12T11:52:33,884 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,884 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,884 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,884 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,884 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,884 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,884 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,884 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,890 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN 2024-11-12T11:52:33,891 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T11:52:33,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,892 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,893 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,902 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,902 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,902 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,902 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9ae3a40ca66d508699f89dac1ef676d7, ASSIGN to state=OFFLINE, location=null, table=test, region=9ae3a40ca66d508699f89dac1ef676d7 to restore RIT 2024-11-12T11:52:33,902 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383902 2024-11-12T11:52:33,902 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,902 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,904 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,905 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,905 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,905 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,910 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,911 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,912 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,912 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,912 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413912 2024-11-12T11:52:33,912 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473912 2024-11-12T11:52:33,912 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-12T11:52:33,912 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,912 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,912 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,912 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,912 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,912 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,912 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,913 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,913 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,915 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,916 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,917 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,917 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,918 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,918 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,918 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,919 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,919 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,919 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,919 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,919 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,919 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,919 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,919 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,919 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN 2024-11-12T11:52:33,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T11:52:33,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,920 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,920 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,930 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=914332156a93810ffdd86211f75b1742, ASSIGN to state=OFFLINE, location=null, table=test, region=914332156a93810ffdd86211f75b1742 to restore RIT 2024-11-12T11:52:33,931 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383931 2024-11-12T11:52:33,931 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,931 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,932 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,934 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,934 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,939 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,940 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,941 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413941 2024-11-12T11:52:33,941 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473941 2024-11-12T11:52:33,941 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-12T11:52:33,941 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,941 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,941 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,941 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,941 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,941 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,941 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,942 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,942 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,943 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,945 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN 2024-11-12T11:52:33,950 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T11:52:33,951 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,951 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,951 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,959 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,959 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,959 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,959 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,960 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,960 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,960 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c34b19c841273fa15336639ee4fe2af6, ASSIGN to state=OFFLINE, location=null, table=test, region=c34b19c841273fa15336639ee4fe2af6 to restore RIT 2024-11-12T11:52:33,960 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383960 2024-11-12T11:52:33,960 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,960 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,962 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,963 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,968 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,969 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,969 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:33,969 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412413969 2024-11-12T11:52:33,969 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412473969 2024-11-12T11:52:33,969 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-12T11:52:33,969 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:33,970 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,970 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:33,970 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:33,970 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:33,970 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:33,970 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:33,970 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:33,971 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:33,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:33,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,973 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:33,974 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:33,975 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,976 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,977 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,977 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:33,978 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN 2024-11-12T11:52:33,978 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T11:52:33,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:33,979 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:33,979 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:33,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:33,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,992 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:33,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:33,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:33,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:33,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:33,993 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=569bd93fc79997920641cb7028f160e0, ASSIGN to state=OFFLINE, location=null, table=test, region=569bd93fc79997920641cb7028f160e0 to restore RIT 2024-11-12T11:52:33,993 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412383993 2024-11-12T11:52:33,994 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:33,994 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,995 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:33,996 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:33,996 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:33,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:33,996 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:33,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:33,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:33,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,002 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,002 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414002 2024-11-12T11:52:34,002 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474002 2024-11-12T11:52:34,002 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-12T11:52:34,002 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,002 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,002 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,003 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,003 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,003 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:34,005 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,008 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,008 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,008 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:34,008 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,008 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN 2024-11-12T11:52:34,008 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T11:52:34,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,009 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,010 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,017 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:34,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:34,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:34,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c874e55257c5357c1082439ee6eda71a, ASSIGN to state=OFFLINE, location=null, table=test, region=c874e55257c5357c1082439ee6eda71a to restore RIT 2024-11-12T11:52:34,018 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384018 2024-11-12T11:52:34,018 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,018 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,020 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,023 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,023 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:34,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,023 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,023 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,027 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,027 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414027 2024-11-12T11:52:34,027 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474027 2024-11-12T11:52:34,027 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-12T11:52:34,027 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,027 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,027 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,027 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,027 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,027 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,027 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,028 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,028 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,031 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,033 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,034 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,034 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,034 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:34,034 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,034 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN 2024-11-12T11:52:34,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T11:52:34,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,035 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,035 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,043 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,046 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37827 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,046 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:34,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:34,047 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37, ASSIGN to state=OFFLINE, location=null, table=test, region=3a9bcab83bd9e8b9783e7f72eb485a37 to restore RIT 2024-11-12T11:52:34,047 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384047 2024-11-12T11:52:34,047 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,047 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,049 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,050 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,050 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:34,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,050 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,053 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,054 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,056 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,056 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414056 2024-11-12T11:52:34,056 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474056 2024-11-12T11:52:34,056 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-12T11:52:34,056 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,056 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,056 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,056 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,057 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,057 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,057 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,057 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,058 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,060 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,062 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,064 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,064 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,064 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,064 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,064 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN 2024-11-12T11:52:34,064 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T11:52:34,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,065 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,065 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,089 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,090 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:34,090 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=57a392190264a7b69a7d25f871aed2c7, ASSIGN to state=OFFLINE, location=null, table=test, region=57a392190264a7b69a7d25f871aed2c7 to restore RIT 2024-11-12T11:52:34,090 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384090 2024-11-12T11:52:34,091 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,091 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,092 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,093 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,093 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,093 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,099 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,100 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,100 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,100 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414100 2024-11-12T11:52:34,100 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474100 2024-11-12T11:52:34,101 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-12T11:52:34,101 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,101 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,101 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,101 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,101 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,101 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,101 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,101 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,102 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,107 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN 2024-11-12T11:52:34,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T11:52:34,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,108 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,108 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,122 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a0b1117abc263d25216a06f7640a6dd0, ASSIGN to state=OFFLINE, location=null, table=test, region=a0b1117abc263d25216a06f7640a6dd0 to restore RIT 2024-11-12T11:52:34,122 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384122 2024-11-12T11:52:34,122 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,122 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,123 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,124 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,124 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,124 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,128 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,129 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,130 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,131 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,131 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414131 2024-11-12T11:52:34,131 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474131 2024-11-12T11:52:34,131 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-12T11:52:34,131 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,132 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,132 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,132 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,132 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,132 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,132 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,133 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,135 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,135 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,136 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,136 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,136 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:34,136 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,136 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,136 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,137 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,138 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,138 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,138 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,138 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T11:52:34,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,139 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,139 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,149 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384149 2024-11-12T11:52:34,149 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,150 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,151 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,153 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,153 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,153 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,154 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,155 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,156 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,156 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,157 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414157 2024-11-12T11:52:34,157 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474157 2024-11-12T11:52:34,157 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-12T11:52:34,157 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,157 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,157 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,157 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,157 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,157 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,158 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:34,160 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,160 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,161 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,161 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,161 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,162 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.1731412349187, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,162 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T11:52:34,163 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,163 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,164 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,171 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,172 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384172 2024-11-12T11:52:34,172 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,172 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,174 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,176 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,176 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,176 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,179 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,179 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414179 2024-11-12T11:52:34,180 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474180 2024-11-12T11:52:34,180 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-12T11:52:34,180 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,180 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,180 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,180 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,180 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,180 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,180 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,180 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,181 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T11:52:34,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,184 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,185 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,185 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,185 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,185 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,185 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b51f50ce4a7c,44417,1731412347465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T11:52:34,186 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,186 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,186 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,197 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384197 2024-11-12T11:52:34,198 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,197 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,199 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,200 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,200 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,200 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,204 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,206 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,206 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,206 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414206 2024-11-12T11:52:34,206 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474206 2024-11-12T11:52:34,206 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-12T11:52:34,206 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,206 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,206 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,206 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,206 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,206 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,206 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,207 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,207 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,209 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T11:52:34,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T11:52:34,210 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,210 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-12T11:52:34,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,211 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,211 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,222 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384222 2024-11-12T11:52:34,222 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,222 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,223 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,224 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,224 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,225 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,230 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,230 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,230 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414230 2024-11-12T11:52:34,230 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474230 2024-11-12T11:52:34,230 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-12T11:52:34,230 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,230 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,231 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,231 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,231 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,231 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,231 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,232 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T11:52:34,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,236 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T11:52:34,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,237 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,238 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,246 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,246 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,246 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,247 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384247 2024-11-12T11:52:34,247 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,247 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,248 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,250 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,250 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,250 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,255 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414255 2024-11-12T11:52:34,255 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474255 2024-11-12T11:52:34,255 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-12T11:52:34,255 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,256 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,256 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,256 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,256 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,256 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T11:52:34,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,261 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure b51f50ce4a7c%2C38813%2C1731412347557.meta.1731412349528.meta, worker=b51f50ce4a7c,34963,1731412347604 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T11:52:34,261 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-12T11:52:34,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,262 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,262 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,272 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384272 2024-11-12T11:52:34,272 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,272 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,274 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,275 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-12T11:52:34,275 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,275 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,280 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,280 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414280 2024-11-12T11:52:34,280 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474280 2024-11-12T11:52:34,280 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-12T11:52:34,280 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T11:52:34,280 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,281 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,281 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,281 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T11:52:34,281 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T11:52:34,281 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T11:52:34,281 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T11:52:34,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T11:52:34,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T11:52:34,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T11:52:34,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-12T11:52:34,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true 2024-11-12T11:52:34,285 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T11:52:34,285 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-12T11:52:34,286 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=61 2024-11-12T11:52:34,286 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=b51f50ce4a7c,44417,1731412347465, table=hbase:meta, region=1588230740 2024-11-12T11:52:34,286 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b51f50ce4a7c,44417,1731412347465, state=OPEN 2024-11-12T11:52:34,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T11:52:34,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T11:52:34,297 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731412384297 2024-11-12T11:52:34,297 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T11:52:34,297 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T11:52:34,299 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-12T11:52:34,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a0b1117abc263d25216a06f7640a6dd0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cbfb2956e6def690689f5e33134c9dd4, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a12d2944d6f25f894da49363450447, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9e98aed23b0727af5f513c5de61a616, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=77, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure c874e55257c5357c1082439ee6eda71a, server=b51f50ce4a7c,44417,1731412347465:java.lang.RuntimeException: inject code bug; ServerCrashProcedure b51f50ce4a7c,38813,1731412347557, splitWal=true, meta=true exec-time=2.7940 sec 2024-11-12T11:52:34,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f786dd883997cd82efafbe7d3d173fc7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=57a392190264a7b69a7d25f871aed2c7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf142f73f167aa11416e3ae10a6bf795, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f3aaa68bd9553e5b9366c15cdfd85491, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9ae3a40ca66d508699f89dac1ef676d7, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88db77d5150c2af8a9c27d5475a321fd, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f828bb7c8aa989003de21e14a812e974, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a9bcab83bd9e8b9783e7f72eb485a37, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=413d5888010bc8c2a41616f921b4b45e, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=44845a612b40b8a8d7da0646602761b7, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d17593aab0898d03d839f7ddc825798, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=914332156a93810ffdd86211f75b1742, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5a4df46f8ab1b85b3e12c8376d94d3d3, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=569bd93fc79997920641cb7028f160e0, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0383ab7f8f8136da6c91952511d8d2e6, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6a247bf888e4a967cccc4510e131fa0f, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c34b19c841273fa15336639ee4fe2af6, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c47427c2053fa4af37c7b70c21c77ef2, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efcf1d7a717f7ea5226c8f259d2cdbe8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c874e55257c5357c1082439ee6eda71a, regionState=OPENING, lastHost=b51f50ce4a7c,38813,1731412347557, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f30fe248b58f81ca59f81dde9d522901, regionState=OPEN, lastHost=b51f50ce4a7c,34963,1731412347604, regionLocation=b51f50ce4a7c,34963,1731412347604, openSeqNum=2 2024-11-12T11:52:34,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b33fbfeb60fca5b204bdf3a8333158e8, regionState=OPEN, lastHost=b51f50ce4a7c,44417,1731412347465, regionLocation=b51f50ce4a7c,44417,1731412347465, openSeqNum=2 2024-11-12T11:52:34,305 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T11:52:34,305 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731412414305 2024-11-12T11:52:34,305 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731412474305 2024-11-12T11:52:34,305 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-12T11:52:34,324 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 381), OpenFileDescriptor=628 (was 651), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=126 (was 119) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1563 (was 1647) 2024-11-12T11:52:34,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T11:52:34,324 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T11:52:34,324 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T11:52:34,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:34,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:34,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T11:52:34,325 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T11:52:34,325 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1211620816, stopped=false 2024-11-12T11:52:34,325 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b51f50ce4a7c,37827,1731412346745 2024-11-12T11:52:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T11:52:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T11:52:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T11:52:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:34,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:34,334 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,334 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T11:52:34,334 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T11:52:34,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:34,334 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T11:52:34,334 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T11:52:34,334 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b51f50ce4a7c,44417,1731412347465' ***** 2024-11-12T11:52:34,335 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T11:52:34,335 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b51f50ce4a7c,34963,1731412347604' ***** 2024-11-12T11:52:34,335 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T11:52:34,335 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T11:52:34,335 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T11:52:34,335 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T11:52:34,335 INFO [RS:2;b51f50ce4a7c:34963 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T11:52:34,335 INFO [RS:2;b51f50ce4a7c:34963 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T11:52:34,335 INFO [RS:0;b51f50ce4a7c:44417 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T11:52:34,335 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T11:52:34,335 INFO [RS:0;b51f50ce4a7c:44417 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T11:52:34,335 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T11:52:34,336 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for 88db77d5150c2af8a9c27d5475a321fd 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for b33fbfeb60fca5b204bdf3a8333158e8 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 44845a612b40b8a8d7da0646602761b7 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for efcf1d7a717f7ea5226c8f259d2cdbe8 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for c874e55257c5357c1082439ee6eda71a 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 57a12d2944d6f25f894da49363450447 2024-11-12T11:52:34,336 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for 413d5888010bc8c2a41616f921b4b45e 2024-11-12T11:52:34,336 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for f30fe248b58f81ca59f81dde9d522901 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 3a9bcab83bd9e8b9783e7f72eb485a37 2024-11-12T11:52:34,336 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for cbfb2956e6def690689f5e33134c9dd4 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for f3aaa68bd9553e5b9366c15cdfd85491 2024-11-12T11:52:34,336 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d17593aab0898d03d839f7ddc825798 2024-11-12T11:52:34,336 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 6a247bf888e4a967cccc4510e131fa0f 2024-11-12T11:52:34,337 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for cf142f73f167aa11416e3ae10a6bf795 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for f786dd883997cd82efafbe7d3d173fc7 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 5a4df46f8ab1b85b3e12c8376d94d3d3 2024-11-12T11:52:34,337 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:34,337 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for c47427c2053fa4af37c7b70c21c77ef2 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 569bd93fc79997920641cb7028f160e0 2024-11-12T11:52:34,337 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(3091): Received CLOSE for 0383ab7f8f8136da6c91952511d8d2e6 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(959): stopping server b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,337 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(959): stopping server b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T11:52:34,337 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T11:52:34,337 INFO [RS:2;b51f50ce4a7c:34963 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b51f50ce4a7c:34963. 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b51f50ce4a7c:44417. 2024-11-12T11:52:34,337 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88db77d5150c2af8a9c27d5475a321fd, disabling compactions & flushes 2024-11-12T11:52:34,337 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:34,337 DEBUG [RS:0;b51f50ce4a7c:44417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T11:52:34,337 DEBUG [RS:2;b51f50ce4a7c:34963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T11:52:34,337 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:34,337 DEBUG [RS:0;b51f50ce4a7c:44417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:34,337 DEBUG [RS:2;b51f50ce4a7c:34963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:34,337 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. after waiting 0 ms 2024-11-12T11:52:34,337 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:34,337 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T11:52:34,337 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T11:52:34,337 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1325): Online Regions={88db77d5150c2af8a9c27d5475a321fd=test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd., 413d5888010bc8c2a41616f921b4b45e=test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e., f30fe248b58f81ca59f81dde9d522901=test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901., cbfb2956e6def690689f5e33134c9dd4=test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4., 2d17593aab0898d03d839f7ddc825798=test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798., cf142f73f167aa11416e3ae10a6bf795=test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795., f9e98aed23b0727af5f513c5de61a616=test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616., c47427c2053fa4af37c7b70c21c77ef2=test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2., 0383ab7f8f8136da6c91952511d8d2e6=test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6.} 2024-11-12T11:52:34,338 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T11:52:34,338 DEBUG [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1351): Waiting on 0383ab7f8f8136da6c91952511d8d2e6, 2d17593aab0898d03d839f7ddc825798, 413d5888010bc8c2a41616f921b4b45e, 88db77d5150c2af8a9c27d5475a321fd, c34b19c841273fa15336639ee4fe2af6, c47427c2053fa4af37c7b70c21c77ef2, cbfb2956e6def690689f5e33134c9dd4, cf142f73f167aa11416e3ae10a6bf795, f30fe248b58f81ca59f81dde9d522901, f9e98aed23b0727af5f513c5de61a616 2024-11-12T11:52:34,338 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b33fbfeb60fca5b204bdf3a8333158e8, disabling compactions & flushes 2024-11-12T11:52:34,338 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:34,338 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:34,338 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-12T11:52:34,338 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. after waiting 0 ms 2024-11-12T11:52:34,338 DEBUG [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3a9bcab83bd9e8b9783e7f72eb485a37, 44845a612b40b8a8d7da0646602761b7, 569bd93fc79997920641cb7028f160e0, 57a12d2944d6f25f894da49363450447, 57a392190264a7b69a7d25f871aed2c7, 5a4df46f8ab1b85b3e12c8376d94d3d3, 6a247bf888e4a967cccc4510e131fa0f, b33fbfeb60fca5b204bdf3a8333158e8, c874e55257c5357c1082439ee6eda71a, efcf1d7a717f7ea5226c8f259d2cdbe8, f3aaa68bd9553e5b9366c15cdfd85491, f786dd883997cd82efafbe7d3d173fc7, f828bb7c8aa989003de21e14a812e974 2024-11-12T11:52:34,338 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:34,338 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T11:52:34,338 INFO [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T11:52:34,338 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T11:52:34,338 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T11:52:34,338 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T11:52:34,339 INFO [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-12T11:52:34,342 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/b33fbfeb60fca5b204bdf3a8333158e8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,343 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/88db77d5150c2af8a9c27d5475a321fd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,345 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:34,345 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:34,345 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88db77d5150c2af8a9c27d5475a321fd: Waiting for close lock at 1731412354336Running coprocessor pre-close hooks at 1731412354337 (+1 ms)Disabling compacts and flushes for region at 1731412354337Disabling writes for close at 1731412354337Writing region close event to WAL at 1731412354338 (+1 ms)Running coprocessor post-close hooks at 1731412354343 (+5 ms)Closed at 1731412354345 (+2 ms) 2024-11-12T11:52:34,345 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b33fbfeb60fca5b204bdf3a8333158e8: Waiting for close lock at 1731412354338Running coprocessor pre-close hooks at 1731412354338Disabling compacts and flushes for region at 1731412354338Disabling writes for close at 1731412354338Writing region close event to WAL at 1731412354338Running coprocessor post-close hooks at 1731412354343 (+5 ms)Closed at 1731412354345 (+2 ms) 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731412349920.b33fbfeb60fca5b204bdf3a8333158e8. 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731412349920.88db77d5150c2af8a9c27d5475a321fd. 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 413d5888010bc8c2a41616f921b4b45e, disabling compactions & flushes 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 44845a612b40b8a8d7da0646602761b7, disabling compactions & flushes 2024-11-12T11:52:34,346 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:34,346 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. after waiting 0 ms 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. after waiting 0 ms 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:34,346 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:34,350 INFO [regionserver/b51f50ce4a7c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T11:52:34,351 INFO [regionserver/b51f50ce4a7c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T11:52:34,352 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/413d5888010bc8c2a41616f921b4b45e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,352 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/44845a612b40b8a8d7da0646602761b7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,353 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 413d5888010bc8c2a41616f921b4b45e: Waiting for close lock at 1731412354346Running coprocessor pre-close hooks at 1731412354346Disabling compacts and flushes for region at 1731412354346Disabling writes for close at 1731412354346Writing region close event to WAL at 1731412354347 (+1 ms)Running coprocessor post-close hooks at 1731412354353 (+6 ms)Closed at 1731412354353 2024-11-12T11:52:34,353 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 44845a612b40b8a8d7da0646602761b7: Waiting for close lock at 1731412354346Running coprocessor pre-close hooks at 1731412354346Disabling compacts and flushes for region at 1731412354346Disabling writes for close at 1731412354346Writing region close event to WAL at 1731412354347 (+1 ms)Running coprocessor post-close hooks at 1731412354353 (+6 ms)Closed at 1731412354353 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731412349920.413d5888010bc8c2a41616f921b4b45e. 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731412349920.44845a612b40b8a8d7da0646602761b7. 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing efcf1d7a717f7ea5226c8f259d2cdbe8, disabling compactions & flushes 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f30fe248b58f81ca59f81dde9d522901, disabling compactions & flushes 2024-11-12T11:52:34,353 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:34,353 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. after waiting 0 ms 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. after waiting 0 ms 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:34,353 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:34,355 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/info/5d5feb2c599d47a284e401d1e18a6d74 is 138, key is test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7./info:regioninfo/1731412353301/Put/seqid=0 2024-11-12T11:52:34,359 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/efcf1d7a717f7ea5226c8f259d2cdbe8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,360 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:34,360 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for efcf1d7a717f7ea5226c8f259d2cdbe8: Waiting for close lock at 1731412354353Running coprocessor pre-close hooks at 1731412354353Disabling compacts and flushes for region at 1731412354353Disabling writes for close at 1731412354353Writing region close event to WAL at 1731412354354 (+1 ms)Running coprocessor post-close hooks at 1731412354360 (+6 ms)Closed at 1731412354360 2024-11-12T11:52:34,360 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731412349920.efcf1d7a717f7ea5226c8f259d2cdbe8. 2024-11-12T11:52:34,360 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c874e55257c5357c1082439ee6eda71a, disabling compactions & flushes 2024-11-12T11:52:34,360 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:34,360 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:34,360 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. after waiting 0 ms 2024-11-12T11:52:34,360 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:34,371 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f30fe248b58f81ca59f81dde9d522901/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,372 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:34,372 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f30fe248b58f81ca59f81dde9d522901: Waiting for close lock at 1731412354353Running coprocessor pre-close hooks at 1731412354353Disabling compacts and flushes for region at 1731412354353Disabling writes for close at 1731412354353Writing region close event to WAL at 1731412354354 (+1 ms)Running coprocessor post-close hooks at 1731412354372 (+18 ms)Closed at 1731412354372 2024-11-12T11:52:34,372 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731412349920.f30fe248b58f81ca59f81dde9d522901. 2024-11-12T11:52:34,373 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cbfb2956e6def690689f5e33134c9dd4, disabling compactions & flushes 2024-11-12T11:52:34,373 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:34,373 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:34,373 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. after waiting 0 ms 2024-11-12T11:52:34,373 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:34,375 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c874e55257c5357c1082439ee6eda71a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,376 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:34,376 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c874e55257c5357c1082439ee6eda71a: Waiting for close lock at 1731412354360Running coprocessor pre-close hooks at 1731412354360Disabling compacts and flushes for region at 1731412354360Disabling writes for close at 1731412354360Writing region close event to WAL at 1731412354363 (+3 ms)Running coprocessor post-close hooks at 1731412354376 (+13 ms)Closed at 1731412354376 2024-11-12T11:52:34,376 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731412349920.c874e55257c5357c1082439ee6eda71a. 2024-11-12T11:52:34,377 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f828bb7c8aa989003de21e14a812e974, disabling compactions & flushes 2024-11-12T11:52:34,377 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:34,377 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:34,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741869_1045 (size=8177) 2024-11-12T11:52:34,377 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. after waiting 0 ms 2024-11-12T11:52:34,377 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:34,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741869_1045 (size=8177) 2024-11-12T11:52:34,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741869_1045 (size=8177) 2024-11-12T11:52:34,379 INFO [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/info/5d5feb2c599d47a284e401d1e18a6d74 2024-11-12T11:52:34,382 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f828bb7c8aa989003de21e14a812e974/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,382 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/cbfb2956e6def690689f5e33134c9dd4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,383 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:34,383 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:34,383 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cbfb2956e6def690689f5e33134c9dd4: Waiting for close lock at 1731412354373Running coprocessor pre-close hooks at 1731412354373Disabling compacts and flushes for region at 1731412354373Disabling writes for close at 1731412354373Writing region close event to WAL at 1731412354375 (+2 ms)Running coprocessor post-close hooks at 1731412354383 (+8 ms)Closed at 1731412354383 2024-11-12T11:52:34,383 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f828bb7c8aa989003de21e14a812e974: Waiting for close lock at 1731412354377Running coprocessor pre-close hooks at 1731412354377Disabling compacts and flushes for region at 1731412354377Disabling writes for close at 1731412354377Writing region close event to WAL at 1731412354378 (+1 ms)Running coprocessor post-close hooks at 1731412354383 (+5 ms)Closed at 1731412354383 2024-11-12T11:52:34,383 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731412349920.cbfb2956e6def690689f5e33134c9dd4. 2024-11-12T11:52:34,383 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731412349920.f828bb7c8aa989003de21e14a812e974. 2024-11-12T11:52:34,384 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 57a12d2944d6f25f894da49363450447, disabling compactions & flushes 2024-11-12T11:52:34,384 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d17593aab0898d03d839f7ddc825798, disabling compactions & flushes 2024-11-12T11:52:34,384 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:34,384 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:34,384 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:34,384 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:34,384 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. after waiting 0 ms 2024-11-12T11:52:34,384 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. after waiting 0 ms 2024-11-12T11:52:34,384 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:34,384 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:34,389 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/57a12d2944d6f25f894da49363450447/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,389 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:34,389 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/.tmp/info/5d5feb2c599d47a284e401d1e18a6d74 as hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/info/5d5feb2c599d47a284e401d1e18a6d74 2024-11-12T11:52:34,390 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 57a12d2944d6f25f894da49363450447: Waiting for close lock at 1731412354383Running coprocessor pre-close hooks at 1731412354383Disabling compacts and flushes for region at 1731412354384 (+1 ms)Disabling writes for close at 1731412354384Writing region close event to WAL at 1731412354384Running coprocessor post-close hooks at 1731412354389 (+5 ms)Closed at 1731412354389 2024-11-12T11:52:34,390 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731412349920.57a12d2944d6f25f894da49363450447. 2024-11-12T11:52:34,390 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3a9bcab83bd9e8b9783e7f72eb485a37, disabling compactions & flushes 2024-11-12T11:52:34,390 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:34,390 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:34,390 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. after waiting 0 ms 2024-11-12T11:52:34,390 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/2d17593aab0898d03d839f7ddc825798/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,390 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:34,391 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:34,391 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d17593aab0898d03d839f7ddc825798: Waiting for close lock at 1731412354384Running coprocessor pre-close hooks at 1731412354384Disabling compacts and flushes for region at 1731412354384Disabling writes for close at 1731412354384Writing region close event to WAL at 1731412354385 (+1 ms)Running coprocessor post-close hooks at 1731412354391 (+6 ms)Closed at 1731412354391 2024-11-12T11:52:34,392 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731412349920.2d17593aab0898d03d839f7ddc825798. 2024-11-12T11:52:34,392 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf142f73f167aa11416e3ae10a6bf795, disabling compactions & flushes 2024-11-12T11:52:34,392 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:34,392 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:34,392 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. after waiting 0 ms 2024-11-12T11:52:34,392 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:34,396 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/3a9bcab83bd9e8b9783e7f72eb485a37/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,396 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/cf142f73f167aa11416e3ae10a6bf795/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,397 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:34,397 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf142f73f167aa11416e3ae10a6bf795: Waiting for close lock at 1731412354392Running coprocessor pre-close hooks at 1731412354392Disabling compacts and flushes for region at 1731412354392Disabling writes for close at 1731412354392Writing region close event to WAL at 1731412354392Running coprocessor post-close hooks at 1731412354397 (+5 ms)Closed at 1731412354397 2024-11-12T11:52:34,397 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731412349920.cf142f73f167aa11416e3ae10a6bf795. 2024-11-12T11:52:34,398 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f9e98aed23b0727af5f513c5de61a616, disabling compactions & flushes 2024-11-12T11:52:34,398 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:34,398 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:34,398 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. after waiting 0 ms 2024-11-12T11:52:34,398 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:34,398 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:34,398 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3a9bcab83bd9e8b9783e7f72eb485a37: Waiting for close lock at 1731412354390Running coprocessor pre-close hooks at 1731412354390Disabling compacts and flushes for region at 1731412354390Disabling writes for close at 1731412354390Writing region close event to WAL at 1731412354391 (+1 ms)Running coprocessor post-close hooks at 1731412354398 (+7 ms)Closed at 1731412354398 2024-11-12T11:52:34,398 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731412349920.3a9bcab83bd9e8b9783e7f72eb485a37. 2024-11-12T11:52:34,398 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f3aaa68bd9553e5b9366c15cdfd85491, disabling compactions & flushes 2024-11-12T11:52:34,399 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:34,399 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:34,399 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. after waiting 0 ms 2024-11-12T11:52:34,399 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:34,400 INFO [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/info/5d5feb2c599d47a284e401d1e18a6d74, entries=24, sequenceid=71, filesize=8.0 K 2024-11-12T11:52:34,401 INFO [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 63ms, sequenceid=71, compaction requested=false 2024-11-12T11:52:34,403 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f9e98aed23b0727af5f513c5de61a616/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,404 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:34,404 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f9e98aed23b0727af5f513c5de61a616: Waiting for close lock at 1731412354398Running coprocessor pre-close hooks at 1731412354398Disabling compacts and flushes for region at 1731412354398Disabling writes for close at 1731412354398Writing region close event to WAL at 1731412354398Running coprocessor post-close hooks at 1731412354404 (+6 ms)Closed at 1731412354404 2024-11-12T11:52:34,404 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731412349920.f9e98aed23b0727af5f513c5de61a616. 2024-11-12T11:52:34,404 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c47427c2053fa4af37c7b70c21c77ef2, disabling compactions & flushes 2024-11-12T11:52:34,404 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:34,404 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:34,404 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. after waiting 0 ms 2024-11-12T11:52:34,404 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:34,407 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f3aaa68bd9553e5b9366c15cdfd85491/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,408 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:34,408 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f3aaa68bd9553e5b9366c15cdfd85491: Waiting for close lock at 1731412354398Running coprocessor pre-close hooks at 1731412354398Disabling compacts and flushes for region at 1731412354398Disabling writes for close at 1731412354399 (+1 ms)Writing region close event to WAL at 1731412354400 (+1 ms)Running coprocessor post-close hooks at 1731412354408 (+8 ms)Closed at 1731412354408 2024-11-12T11:52:34,408 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731412349920.f3aaa68bd9553e5b9366c15cdfd85491. 2024-11-12T11:52:34,408 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6a247bf888e4a967cccc4510e131fa0f, disabling compactions & flushes 2024-11-12T11:52:34,408 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:34,408 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:34,408 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. after waiting 0 ms 2024-11-12T11:52:34,409 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:34,414 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/6a247bf888e4a967cccc4510e131fa0f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,414 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/c47427c2053fa4af37c7b70c21c77ef2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,414 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T11:52:34,415 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:34,415 INFO [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c47427c2053fa4af37c7b70c21c77ef2: Waiting for close lock at 1731412354404Running coprocessor pre-close hooks at 1731412354404Disabling compacts and flushes for region at 1731412354404Disabling writes for close at 1731412354404Writing region close event to WAL at 1731412354410 (+6 ms)Running coprocessor post-close hooks at 1731412354415 (+5 ms)Closed at 1731412354415 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731412354338Running coprocessor pre-close hooks at 1731412354338Disabling compacts and flushes for region at 1731412354338Disabling writes for close at 1731412354338Obtaining lock to block concurrent updates at 1731412354339 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731412354339Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1731412354347 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731412354348 (+1 ms)Flushing 1588230740/info: creating writer at 1731412354349 (+1 ms)Flushing 1588230740/info: appending metadata at 1731412354354 (+5 ms)Flushing 1588230740/info: closing flushed file at 1731412354354Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@669e1e3c: reopening flushed file at 1731412354388 (+34 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 63ms, sequenceid=71, compaction requested=false at 1731412354401 (+13 ms)Writing region close event to WAL at 1731412354410 (+9 ms)Running coprocessor post-close hooks at 1731412354414 (+4 ms)Closed at 1731412354415 (+1 ms) 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_META-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731412349920.c47427c2053fa4af37c7b70c21c77ef2. 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0383ab7f8f8136da6c91952511d8d2e6, disabling compactions & flushes 2024-11-12T11:52:34,415 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:34,415 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6a247bf888e4a967cccc4510e131fa0f: Waiting for close lock at 1731412354408Running coprocessor pre-close hooks at 1731412354408Disabling compacts and flushes for region at 1731412354408Disabling writes for close at 1731412354409 (+1 ms)Writing region close event to WAL at 1731412354410 (+1 ms)Running coprocessor post-close hooks at 1731412354415 (+5 ms)Closed at 1731412354415 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731412349920.6a247bf888e4a967cccc4510e131fa0f. 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. after waiting 0 ms 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f786dd883997cd82efafbe7d3d173fc7, disabling compactions & flushes 2024-11-12T11:52:34,415 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. after waiting 0 ms 2024-11-12T11:52:34,415 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:34,420 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/f786dd883997cd82efafbe7d3d173fc7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,420 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/0383ab7f8f8136da6c91952511d8d2e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,420 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:34,420 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f786dd883997cd82efafbe7d3d173fc7: Waiting for close lock at 1731412354415Running coprocessor pre-close hooks at 1731412354415Disabling compacts and flushes for region at 1731412354415Disabling writes for close at 1731412354415Writing region close event to WAL at 1731412354416 (+1 ms)Running coprocessor post-close hooks at 1731412354420 (+4 ms)Closed at 1731412354420 2024-11-12T11:52:34,420 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:34,421 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731412349920.f786dd883997cd82efafbe7d3d173fc7. 2024-11-12T11:52:34,421 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0383ab7f8f8136da6c91952511d8d2e6: Waiting for close lock at 1731412354415Running coprocessor pre-close hooks at 1731412354415Disabling compacts and flushes for region at 1731412354415Disabling writes for close at 1731412354415Writing region close event to WAL at 1731412354416 (+1 ms)Running coprocessor post-close hooks at 1731412354420 (+4 ms)Closed at 1731412354420 2024-11-12T11:52:34,421 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5a4df46f8ab1b85b3e12c8376d94d3d3, disabling compactions & flushes 2024-11-12T11:52:34,421 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731412349920.0383ab7f8f8136da6c91952511d8d2e6. 2024-11-12T11:52:34,421 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:34,421 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:34,421 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. after waiting 0 ms 2024-11-12T11:52:34,421 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:34,425 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/5a4df46f8ab1b85b3e12c8376d94d3d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,425 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:34,426 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5a4df46f8ab1b85b3e12c8376d94d3d3: Waiting for close lock at 1731412354421Running coprocessor pre-close hooks at 1731412354421Disabling compacts and flushes for region at 1731412354421Disabling writes for close at 1731412354421Writing region close event to WAL at 1731412354421Running coprocessor post-close hooks at 1731412354425 (+4 ms)Closed at 1731412354425 2024-11-12T11:52:34,426 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731412349920.5a4df46f8ab1b85b3e12c8376d94d3d3. 2024-11-12T11:52:34,426 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 569bd93fc79997920641cb7028f160e0, disabling compactions & flushes 2024-11-12T11:52:34,426 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:34,426 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:34,426 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. after waiting 0 ms 2024-11-12T11:52:34,426 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:34,430 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/data/default/test/569bd93fc79997920641cb7028f160e0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T11:52:34,431 INFO [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:34,431 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 569bd93fc79997920641cb7028f160e0: Waiting for close lock at 1731412354426Running coprocessor pre-close hooks at 1731412354426Disabling compacts and flushes for region at 1731412354426Disabling writes for close at 1731412354426Writing region close event to WAL at 1731412354426Running coprocessor post-close hooks at 1731412354431 (+5 ms)Closed at 1731412354431 2024-11-12T11:52:34,431 DEBUG [RS_CLOSE_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731412349920.569bd93fc79997920641cb7028f160e0. 2024-11-12T11:52:34,531 WARN [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7., aborting... java.io.IOException: Failed to report opened region to master: test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T11:52:34,532 ERROR [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2440): ***** ABORTING region server b51f50ce4a7c,44417,1731412347465: Failed to open region test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,fff,1731412349920.57a392190264a7b69a7d25f871aed2c7. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T11:52:34,532 ERROR [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T11:52:34,532 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T11:52:34,533 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T11:52:34,533 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T11:52:34,534 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T11:52:34,534 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 454269640 }, "NonHeapMemoryUsage": { "committed": 161218560, "init": 7667712, "max": -1, "used": 158506528 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ] } 2024-11-12T11:52:34,536 WARN [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2470): Unable to report fatal error to master org.apache.hbase.thirdparty.com.google.protobuf.ServiceException: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=b51f50ce4a7c:37827 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:344) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportRSFatalError(RegionServerStatusProtos.java:17290) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.abort(HRegionServer.java:2467) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abortRegionServer(SingleProcessHBaseCluster.java:192) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$2.run(SingleProcessHBaseCluster.java:185) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.abort(SingleProcessHBaseCluster.java:182) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.handleException(AssignRegionHandler.java:173) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:106) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=b51f50ce4a7c:37827 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 15 more 2024-11-12T11:52:34,538 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(976): stopping server b51f50ce4a7c,34963,1731412347604; all regions closed. 2024-11-12T11:52:34,538 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(976): stopping server b51f50ce4a7c,44417,1731412347465; all regions closed. 2024-11-12T11:52:34,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741865_1041 (size=4192) 2024-11-12T11:52:34,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741835_1011 (size=5194) 2024-11-12T11:52:34,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741835_1011 (size=5194) 2024-11-12T11:52:34,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741865_1041 (size=4192) 2024-11-12T11:52:34,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741865_1041 (size=4192) 2024-11-12T11:52:34,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741835_1011 (size=5194) 2024-11-12T11:52:34,546 DEBUG [RS:2;b51f50ce4a7c:34963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/oldWALs 2024-11-12T11:52:34,547 INFO [RS:2;b51f50ce4a7c:34963 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b51f50ce4a7c%2C34963%2C1731412347604:(num 1731412349187) 2024-11-12T11:52:34,547 DEBUG [RS:2;b51f50ce4a7c:34963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:34,547 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T11:52:34,547 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T11:52:34,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741833_1009 (size=6805) 2024-11-12T11:52:34,547 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/WALs/b51f50ce4a7c,44417,1731412347465/b51f50ce4a7c%2C44417%2C1731412347465.1731412349187 not finished, retry = 0 2024-11-12T11:52:34,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741833_1009 (size=6805) 2024-11-12T11:52:34,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741833_1009 (size=6805) 2024-11-12T11:52:34,547 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.ChoreService(370): Chore service for: regionserver/b51f50ce4a7c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T11:52:34,548 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T11:52:34,548 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T11:52:34,548 INFO [regionserver/b51f50ce4a7c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T11:52:34,548 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T11:52:34,548 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T11:52:34,548 WARN [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T11:52:34,548 WARN [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6., aborting... java.io.IOException: Failed to report opened region to master: test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T11:52:34,549 ERROR [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2440): ***** ABORTING region server b51f50ce4a7c,34963,1731412347604: Failed to open region test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,uuu,1731412349920.c34b19c841273fa15336639ee4fe2af6. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T11:52:34,549 ERROR [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T11:52:34,549 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T11:52:34,549 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T11:52:34,550 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T11:52:34,550 DEBUG [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T11:52:34,550 INFO [RS_OPEN_REGION-regionserver/b51f50ce4a7c:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 456366792 }, "NonHeapMemoryUsage": { "committed": 161284096, "init": 7667712, "max": -1, "used": 158617544 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "b51f50ce4a7c" } ] } 2024-11-12T11:52:34,550 INFO [RS:2;b51f50ce4a7c:34963 {}] executor.ExecutorService(108): Executor-10-RS_OPEN_REGION-regionserver/b51f50ce4a7c:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-12T11:52:34,550 INFO [RS:2;b51f50ce4a7c:34963 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34963 2024-11-12T11:52:34,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T11:52:34,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b51f50ce4a7c,34963,1731412347604 2024-11-12T11:52:34,558 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T11:52:34,559 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b51f50ce4a7c,34963,1731412347604] 2024-11-12T11:52:34,575 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b51f50ce4a7c,34963,1731412347604 already deleted, retry=false 2024-11-12T11:52:34,575 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b51f50ce4a7c,34963,1731412347604 expired; onlineServers=1 2024-11-12T11:52:34,649 DEBUG [RS:0;b51f50ce4a7c:44417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T11:52:34,649 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T11:52:34,649 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T11:52:34,649 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.ChoreService(370): Chore service for: regionserver/b51f50ce4a7c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T11:52:34,649 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T11:52:34,649 INFO [regionserver/b51f50ce4a7c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T11:52:34,650 INFO [RS:0;b51f50ce4a7c:44417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44417 2024-11-12T11:52:34,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T11:52:34,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b51f50ce4a7c,44417,1731412347465 2024-11-12T11:52:34,659 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T11:52:34,660 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b51f50ce4a7c,44417,1731412347465] 2024-11-12T11:52:34,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T11:52:34,667 INFO [RS:2;b51f50ce4a7c:34963 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T11:52:34,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34963-0x101302d66c30003, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T11:52:34,667 INFO [RS:2;b51f50ce4a7c:34963 {}] regionserver.HRegionServer(1031): Exiting; stopping=b51f50ce4a7c,34963,1731412347604; zookeeper connection closed. 2024-11-12T11:52:34,667 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@416eb25b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@416eb25b 2024-11-12T11:52:34,675 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b51f50ce4a7c,44417,1731412347465 already deleted, retry=false 2024-11-12T11:52:34,675 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b51f50ce4a7c,44417,1731412347465 expired; onlineServers=0 2024-11-12T11:52:34,675 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b51f50ce4a7c,37827,1731412346745' ***** 2024-11-12T11:52:34,675 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T11:52:34,675 INFO [M:0;b51f50ce4a7c:37827 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T11:52:34,676 INFO [M:0;b51f50ce4a7c:37827 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T11:52:34,676 DEBUG [M:0;b51f50ce4a7c:37827 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T11:52:34,676 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T11:52:34,676 DEBUG [M:0;b51f50ce4a7c:37827 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T11:52:34,676 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster-HFileCleaner.large.0-1731412348758 {}] cleaner.HFileCleaner(306): Exit Thread[master/b51f50ce4a7c:0:becomeActiveMaster-HFileCleaner.large.0-1731412348758,5,FailOnTimeoutGroup] 2024-11-12T11:52:34,676 DEBUG [master/b51f50ce4a7c:0:becomeActiveMaster-HFileCleaner.small.0-1731412348762 {}] cleaner.HFileCleaner(306): Exit Thread[master/b51f50ce4a7c:0:becomeActiveMaster-HFileCleaner.small.0-1731412348762,5,FailOnTimeoutGroup] 2024-11-12T11:52:34,676 INFO [M:0;b51f50ce4a7c:37827 {}] hbase.ChoreService(370): Chore service for: master/b51f50ce4a7c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T11:52:34,676 INFO [M:0;b51f50ce4a7c:37827 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T11:52:34,676 DEBUG [M:0;b51f50ce4a7c:37827 {}] master.HMaster(1795): Stopping service threads 2024-11-12T11:52:34,676 INFO [M:0;b51f50ce4a7c:37827 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T11:52:34,677 INFO [M:0;b51f50ce4a7c:37827 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T11:52:34,677 INFO [M:0;b51f50ce4a7c:37827 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T11:52:34,677 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T11:52:34,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T11:52:34,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T11:52:34,684 DEBUG [M:0;b51f50ce4a7c:37827 {}] zookeeper.ZKUtil(347): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T11:52:34,684 WARN [M:0;b51f50ce4a7c:37827 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T11:52:34,684 INFO [M:0;b51f50ce4a7c:37827 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/.lastflushedseqids 2024-11-12T11:52:34,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741870_1046 (size=1) 2024-11-12T11:52:34,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741870_1046 (size=1) 2024-11-12T11:52:34,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741870_1046 (size=1) 2024-11-12T11:52:34,701 INFO [M:0;b51f50ce4a7c:37827 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T11:52:34,701 INFO [M:0;b51f50ce4a7c:37827 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T11:52:34,701 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T11:52:34,701 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T11:52:34,701 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T11:52:34,701 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T11:52:34,701 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T11:52:34,701 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.41 KB heapSize=301.89 KB 2024-11-12T11:52:34,717 DEBUG [M:0;b51f50ce4a7c:37827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ace2d1c371d4b02bbcf85168dbea1d4 is 82, key is hbase:meta,,1/info:regioninfo/1731412352864/Put/seqid=0 2024-11-12T11:52:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741871_1047 (size=6071) 2024-11-12T11:52:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741871_1047 (size=6071) 2024-11-12T11:52:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741871_1047 (size=6071) 2024-11-12T11:52:34,725 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ace2d1c371d4b02bbcf85168dbea1d4 2024-11-12T11:52:34,749 DEBUG [M:0;b51f50ce4a7c:37827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/872ab0d9d6f544b0a33ce6560694cac1 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731412354299/Put/seqid=0 2024-11-12T11:52:34,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741872_1048 (size=12799) 2024-11-12T11:52:34,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741872_1048 (size=12799) 2024-11-12T11:52:34,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741872_1048 (size=12799) 2024-11-12T11:52:34,756 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=247.14 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/872ab0d9d6f544b0a33ce6560694cac1 2024-11-12T11:52:34,767 INFO [RS:0;b51f50ce4a7c:44417 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T11:52:34,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T11:52:34,767 INFO [RS:0;b51f50ce4a7c:44417 {}] regionserver.HRegionServer(1031): Exiting; stopping=b51f50ce4a7c,44417,1731412347465; zookeeper connection closed. 2024-11-12T11:52:34,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101302d66c30001, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T11:52:34,767 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f1a5dbd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f1a5dbd 2024-11-12T11:52:34,768 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-12T11:52:34,780 DEBUG [M:0;b51f50ce4a7c:37827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a878bef003e4596be7719ae780341e1 is 69, key is b51f50ce4a7c,34963,1731412347604/rs:state/1731412348913/Put/seqid=0 2024-11-12T11:52:34,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741873_1049 (size=5447) 2024-11-12T11:52:34,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741873_1049 (size=5447) 2024-11-12T11:52:34,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741873_1049 (size=5447) 2024-11-12T11:52:34,788 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a878bef003e4596be7719ae780341e1 2024-11-12T11:52:34,795 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1a878bef003e4596be7719ae780341e1 2024-11-12T11:52:34,811 DEBUG [M:0;b51f50ce4a7c:37827 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e2afc7d7d8f74219b53a6b5e1038da5e is 52, key is load_balancer_on/state:d/1731412351270/Put/seqid=0 2024-11-12T11:52:34,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741874_1050 (size=5058) 2024-11-12T11:52:34,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741874_1050 (size=5058) 2024-11-12T11:52:34,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741874_1050 (size=5058) 2024-11-12T11:52:34,818 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e2afc7d7d8f74219b53a6b5e1038da5e 2024-11-12T11:52:34,825 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4ace2d1c371d4b02bbcf85168dbea1d4 as hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4ace2d1c371d4b02bbcf85168dbea1d4 2024-11-12T11:52:34,832 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4ace2d1c371d4b02bbcf85168dbea1d4, entries=14, sequenceid=570, filesize=5.9 K 2024-11-12T11:52:34,833 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/872ab0d9d6f544b0a33ce6560694cac1 as hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/872ab0d9d6f544b0a33ce6560694cac1 2024-11-12T11:52:34,840 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/872ab0d9d6f544b0a33ce6560694cac1, entries=79, sequenceid=570, filesize=12.5 K 2024-11-12T11:52:34,841 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1a878bef003e4596be7719ae780341e1 as hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1a878bef003e4596be7719ae780341e1 2024-11-12T11:52:34,849 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1a878bef003e4596be7719ae780341e1 2024-11-12T11:52:34,849 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1a878bef003e4596be7719ae780341e1, entries=3, sequenceid=570, filesize=5.3 K 2024-11-12T11:52:34,851 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e2afc7d7d8f74219b53a6b5e1038da5e as hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e2afc7d7d8f74219b53a6b5e1038da5e 2024-11-12T11:52:34,857 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42603/user/jenkins/test-data/ce57d486-deea-046b-8923-a479623a4b09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e2afc7d7d8f74219b53a6b5e1038da5e, entries=1, sequenceid=570, filesize=4.9 K 2024-11-12T11:52:34,859 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.41 KB/254375, heapSize ~301.83 KB/309072, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 158ms, sequenceid=570, compaction requested=false 2024-11-12T11:52:34,860 INFO [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T11:52:34,860 DEBUG [M:0;b51f50ce4a7c:37827 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731412354701Disabling compacts and flushes for region at 1731412354701Disabling writes for close at 1731412354701Obtaining lock to block concurrent updates at 1731412354701Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731412354701Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254375, getHeapSize=309072, getOffHeapSize=0, getCellsCount=709 at 1731412354702 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731412354702Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731412354702Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731412354717 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731412354717Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731412354731 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731412354748 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731412354748Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731412354763 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731412354779 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731412354779Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731412354795 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731412354810 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731412354810Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32c80bf2: reopening flushed file at 1731412354824 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6593bc1c: reopening flushed file at 1731412354832 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@560a5b8c: reopening flushed file at 1731412354840 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@183689e2: reopening flushed file at 1731412354850 (+10 ms)Finished flush of dataSize ~248.41 KB/254375, heapSize ~301.83 KB/309072, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 158ms, sequenceid=570, compaction requested=false at 1731412354859 (+9 ms)Writing region close event to WAL at 1731412354860 (+1 ms)Closed at 1731412354860 2024-11-12T11:52:34,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42055 is added to blk_1073741830_1006 (size=292434) 2024-11-12T11:52:34,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44441 is added to blk_1073741830_1006 (size=292434) 2024-11-12T11:52:34,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39411 is added to blk_1073741830_1006 (size=292434) 2024-11-12T11:52:34,865 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T11:52:34,865 INFO [M:0;b51f50ce4a7c:37827 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T11:52:34,865 INFO [M:0;b51f50ce4a7c:37827 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37827 2024-11-12T11:52:34,865 INFO [M:0;b51f50ce4a7c:37827 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T11:52:34,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T11:52:34,975 INFO [M:0;b51f50ce4a7c:37827 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T11:52:34,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37827-0x101302d66c30000, quorum=127.0.0.1:60801, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T11:52:35,015 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T11:52:35,016 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T11:52:35,017 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T11:52:35,017 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T11:52:35,017 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir/,STOPPED} 2024-11-12T11:52:35,019 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T11:52:35,019 WARN [BP-1490150824-172.17.0.2-1731412343017 heartbeating to localhost/127.0.0.1:42603 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T11:52:35,019 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T11:52:35,020 WARN [BP-1490150824-172.17.0.2-1731412343017 heartbeating to localhost/127.0.0.1:42603 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1490150824-172.17.0.2-1731412343017 (Datanode Uuid 16c2faf6-3c7c-4efd-823f-54c628db37f6) service to localhost/127.0.0.1:42603 2024-11-12T11:52:35,022 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data5/current/BP-1490150824-172.17.0.2-1731412343017 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T11:52:35,022 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data6/current/BP-1490150824-172.17.0.2-1731412343017 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T11:52:35,023 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T11:52:35,026 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T11:52:35,026 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T11:52:35,026 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T11:52:35,027 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T11:52:35,027 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir/,STOPPED} 2024-11-12T11:52:35,028 WARN [BP-1490150824-172.17.0.2-1731412343017 heartbeating to localhost/127.0.0.1:42603 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T11:52:35,028 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T11:52:35,028 WARN [BP-1490150824-172.17.0.2-1731412343017 heartbeating to localhost/127.0.0.1:42603 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1490150824-172.17.0.2-1731412343017 (Datanode Uuid 8dfd5a13-6277-405e-af3b-99df7f0e8c31) service to localhost/127.0.0.1:42603 2024-11-12T11:52:35,028 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T11:52:35,029 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data3/current/BP-1490150824-172.17.0.2-1731412343017 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T11:52:35,030 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data4/current/BP-1490150824-172.17.0.2-1731412343017 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T11:52:35,030 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T11:52:35,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T11:52:35,032 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T11:52:35,032 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T11:52:35,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T11:52:35,033 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir/,STOPPED} 2024-11-12T11:52:35,034 WARN [BP-1490150824-172.17.0.2-1731412343017 heartbeating to localhost/127.0.0.1:42603 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T11:52:35,034 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T11:52:35,034 WARN [BP-1490150824-172.17.0.2-1731412343017 heartbeating to localhost/127.0.0.1:42603 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1490150824-172.17.0.2-1731412343017 (Datanode Uuid 1fc17cea-dfae-4682-9c7c-d59a9b861595) service to localhost/127.0.0.1:42603 2024-11-12T11:52:35,034 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T11:52:35,034 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data1/current/BP-1490150824-172.17.0.2-1731412343017 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T11:52:35,034 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/cluster_37f7331a-834e-5d31-6fe5-00e6e65ef680/data/data2/current/BP-1490150824-172.17.0.2-1731412343017 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T11:52:35,034 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T11:52:35,045 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T11:52:35,046 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T11:52:35,046 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T11:52:35,046 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T11:52:35,046 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1b4fd9e1-84d3-1078-4f8d-48672703f305/hadoop.log.dir/,STOPPED} 2024-11-12T11:52:35,055 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T11:52:35,077 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T11:52:35,083 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-12T11:52:35,107 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down