2024-11-26 20:13:55,723 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b 2024-11-26 20:13:55,744 main DEBUG Took 0.017268 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 20:13:55,745 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 20:13:55,745 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 20:13:55,748 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 20:13:55,750 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,788 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 20:13:55,813 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,824 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,826 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,828 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,830 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,832 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,834 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,836 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,838 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,839 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,842 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,843 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,844 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,845 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 20:13:55,846 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,846 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 20:13:55,848 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 20:13:55,851 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 20:13:55,861 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 20:13:55,862 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 20:13:55,867 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 20:13:55,867 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 20:13:55,883 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 20:13:55,887 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 20:13:55,890 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 20:13:55,891 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 20:13:55,891 main DEBUG createAppenders(={Console}) 2024-11-26 20:13:55,892 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b initialized 2024-11-26 20:13:55,893 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b 2024-11-26 20:13:55,894 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@72c8e7b OK. 2024-11-26 20:13:55,895 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 20:13:55,896 main DEBUG OutputStream closed 2024-11-26 20:13:55,896 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 20:13:55,897 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 20:13:55,897 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5d0a1059 OK 2024-11-26 20:13:56,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 20:13:56,042 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 20:13:56,044 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 20:13:56,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 20:13:56,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 20:13:56,047 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 20:13:56,047 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 20:13:56,048 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 20:13:56,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 20:13:56,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 20:13:56,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 20:13:56,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 20:13:56,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 20:13:56,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 20:13:56,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 20:13:56,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 20:13:56,054 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 20:13:56,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 20:13:56,059 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 20:13:56,060 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@539d019) with optional ClassLoader: null 2024-11-26 20:13:56,060 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 20:13:56,062 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@539d019] started OK. 2024-11-26T20:13:56,084 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-26 20:13:56,089 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 20:13:56,089 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T20:13:56,683 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d 2024-11-26T20:13:56,714 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T20:13:56,741 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1, deleteOnExit=true 2024-11-26T20:13:56,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T20:13:56,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/test.cache.data in system properties and HBase conf 2024-11-26T20:13:56,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T20:13:56,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir in system properties and HBase conf 2024-11-26T20:13:56,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T20:13:56,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T20:13:56,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T20:13:56,957 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T20:13:57,222 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T20:13:57,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T20:13:57,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T20:13:57,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T20:13:57,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T20:13:57,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T20:13:57,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T20:13:57,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T20:13:57,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T20:13:57,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T20:13:57,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/nfs.dump.dir in system properties and HBase conf 2024-11-26T20:13:57,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/java.io.tmpdir in system properties and HBase conf 2024-11-26T20:13:57,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T20:13:57,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T20:13:57,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T20:14:08,622 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T20:14:08,687 INFO [Time-limited test {}] log.Log(170): Logging initialized @14154ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T20:14:08,753 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T20:14:08,809 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T20:14:08,828 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T20:14:08,828 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T20:14:08,830 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T20:14:08,841 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T20:14:08,844 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir/,AVAILABLE} 2024-11-26T20:14:08,845 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T20:14:09,046 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/java.io.tmpdir/jetty-localhost-38503-hadoop-hdfs-3_4_1-tests_jar-_-any-17638639109964934406/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T20:14:09,052 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:38503} 2024-11-26T20:14:09,053 INFO [Time-limited test {}] server.Server(415): Started @14520ms 2024-11-26T20:14:09,721 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T20:14:09,728 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T20:14:09,729 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T20:14:09,729 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T20:14:09,729 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T20:14:09,730 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir/,AVAILABLE} 2024-11-26T20:14:09,731 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T20:14:09,831 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/java.io.tmpdir/jetty-localhost-44269-hadoop-hdfs-3_4_1-tests_jar-_-any-12876911132594546681/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T20:14:09,831 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:44269} 2024-11-26T20:14:09,832 INFO [Time-limited test {}] server.Server(415): Started @15299ms 2024-11-26T20:14:09,879 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T20:14:09,994 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T20:14:10,002 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T20:14:10,006 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T20:14:10,006 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T20:14:10,006 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T20:14:10,008 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir/,AVAILABLE} 2024-11-26T20:14:10,008 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T20:14:10,114 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/java.io.tmpdir/jetty-localhost-39985-hadoop-hdfs-3_4_1-tests_jar-_-any-6758823628152245220/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T20:14:10,115 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:39985} 2024-11-26T20:14:10,115 INFO [Time-limited test {}] server.Server(415): Started @15582ms 2024-11-26T20:14:10,117 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T20:14:10,157 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T20:14:10,162 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T20:14:10,164 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T20:14:10,164 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T20:14:10,164 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T20:14:10,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir/,AVAILABLE} 2024-11-26T20:14:10,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T20:14:10,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/java.io.tmpdir/jetty-localhost-46455-hadoop-hdfs-3_4_1-tests_jar-_-any-16999252453603576399/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T20:14:10,267 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:46455} 2024-11-26T20:14:10,267 INFO [Time-limited test {}] server.Server(415): Started @15734ms 2024-11-26T20:14:10,269 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T20:14:12,284 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data4/current/BP-1003607189-172.17.0.3-1732652041092/current, will proceed with Du for space computation calculation, 2024-11-26T20:14:12,284 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data3/current/BP-1003607189-172.17.0.3-1732652041092/current, will proceed with Du for space computation calculation, 2024-11-26T20:14:12,291 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data1/current/BP-1003607189-172.17.0.3-1732652041092/current, will proceed with Du for space computation calculation, 2024-11-26T20:14:12,293 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data2/current/BP-1003607189-172.17.0.3-1732652041092/current, will proceed with Du for space computation calculation, 2024-11-26T20:14:12,307 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data5/current/BP-1003607189-172.17.0.3-1732652041092/current, will proceed with Du for space computation calculation, 2024-11-26T20:14:12,325 WARN [Thread-131 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data6/current/BP-1003607189-172.17.0.3-1732652041092/current, will proceed with Du for space computation calculation, 2024-11-26T20:14:12,448 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T20:14:12,471 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T20:14:12,533 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T20:14:12,575 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcff912a5bee8fa29 with lease ID 0xf0e3c9b39b08bfc: Processing first storage report for DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9 from datanode DatanodeRegistration(127.0.0.1:45553, datanodeUuid=36f9bb8a-7dcf-4d56-8314-0ba5daeef0f4, infoPort=37737, infoSecurePort=0, ipcPort=46249, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092) 2024-11-26T20:14:12,577 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcff912a5bee8fa29 with lease ID 0xf0e3c9b39b08bfc: from storage DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9 node DatanodeRegistration(127.0.0.1:45553, datanodeUuid=36f9bb8a-7dcf-4d56-8314-0ba5daeef0f4, infoPort=37737, infoSecurePort=0, ipcPort=46249, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-26T20:14:12,578 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xee2856a39b26831d with lease ID 0xf0e3c9b39b08bfa: Processing first storage report for DS-ebeab9c5-a6a6-4986-877c-eaff3583f349 from datanode DatanodeRegistration(127.0.0.1:36117, datanodeUuid=36b267de-6be8-493c-ae7d-2084bce59074, infoPort=46547, infoSecurePort=0, ipcPort=33517, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092) 2024-11-26T20:14:12,579 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xee2856a39b26831d with lease ID 0xf0e3c9b39b08bfa: from storage DS-ebeab9c5-a6a6-4986-877c-eaff3583f349 node DatanodeRegistration(127.0.0.1:36117, datanodeUuid=36b267de-6be8-493c-ae7d-2084bce59074, infoPort=46547, infoSecurePort=0, ipcPort=33517, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T20:14:12,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xddcc0e04544e925d with lease ID 0xf0e3c9b39b08bfb: Processing first storage report for DS-a3350620-7d44-4208-a51a-24cd0df60136 from datanode DatanodeRegistration(127.0.0.1:34443, datanodeUuid=3d7e7ac9-d193-4317-a686-8f41042da5ab, infoPort=33291, infoSecurePort=0, ipcPort=46351, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092) 2024-11-26T20:14:12,580 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xddcc0e04544e925d with lease ID 0xf0e3c9b39b08bfb: from storage DS-a3350620-7d44-4208-a51a-24cd0df60136 node DatanodeRegistration(127.0.0.1:34443, datanodeUuid=3d7e7ac9-d193-4317-a686-8f41042da5ab, infoPort=33291, infoSecurePort=0, ipcPort=46351, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T20:14:12,580 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xee2856a39b26831d with lease ID 0xf0e3c9b39b08bfa: Processing first storage report for DS-ed51968a-eb3f-4eb6-8890-e1d6eec8349d from datanode DatanodeRegistration(127.0.0.1:36117, datanodeUuid=36b267de-6be8-493c-ae7d-2084bce59074, infoPort=46547, infoSecurePort=0, ipcPort=33517, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092) 2024-11-26T20:14:12,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xee2856a39b26831d with lease ID 0xf0e3c9b39b08bfa: from storage DS-ed51968a-eb3f-4eb6-8890-e1d6eec8349d node DatanodeRegistration(127.0.0.1:36117, datanodeUuid=36b267de-6be8-493c-ae7d-2084bce59074, infoPort=46547, infoSecurePort=0, ipcPort=33517, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T20:14:12,581 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xddcc0e04544e925d with lease ID 0xf0e3c9b39b08bfb: Processing first storage report for DS-d4a43b77-e6d1-4fcf-a974-a65f9d2ab9fe from datanode DatanodeRegistration(127.0.0.1:34443, datanodeUuid=3d7e7ac9-d193-4317-a686-8f41042da5ab, infoPort=33291, infoSecurePort=0, ipcPort=46351, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092) 2024-11-26T20:14:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xddcc0e04544e925d with lease ID 0xf0e3c9b39b08bfb: from storage DS-d4a43b77-e6d1-4fcf-a974-a65f9d2ab9fe node DatanodeRegistration(127.0.0.1:34443, datanodeUuid=3d7e7ac9-d193-4317-a686-8f41042da5ab, infoPort=33291, infoSecurePort=0, ipcPort=46351, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T20:14:12,596 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcff912a5bee8fa29 with lease ID 0xf0e3c9b39b08bfc: Processing first storage report for DS-d28f6e31-8777-4f8f-91f1-b65fc4657b3d from datanode DatanodeRegistration(127.0.0.1:45553, datanodeUuid=36f9bb8a-7dcf-4d56-8314-0ba5daeef0f4, infoPort=37737, infoSecurePort=0, ipcPort=46249, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092) 2024-11-26T20:14:12,597 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcff912a5bee8fa29 with lease ID 0xf0e3c9b39b08bfc: from storage DS-d28f6e31-8777-4f8f-91f1-b65fc4657b3d node DatanodeRegistration(127.0.0.1:45553, datanodeUuid=36f9bb8a-7dcf-4d56-8314-0ba5daeef0f4, infoPort=37737, infoSecurePort=0, ipcPort=46249, storageInfo=lv=-57;cid=testClusterID;nsid=2039643385;c=1732652041092), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T20:14:12,627 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d 2024-11-26T20:14:12,844 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/zookeeper_0, clientPort=50191, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T20:14:12,872 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50191 2024-11-26T20:14:12,914 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:12,920 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:13,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741825_1001 (size=7) 2024-11-26T20:14:13,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741825_1001 (size=7) 2024-11-26T20:14:13,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741825_1001 (size=7) 2024-11-26T20:14:13,570 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d with version=8 2024-11-26T20:14:13,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/hbase-staging 2024-11-26T20:14:13,738 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T20:14:14,093 INFO [Time-limited test {}] client.ConnectionUtils(128): master/76919af20f29:0 server-side Connection retries=45 2024-11-26T20:14:14,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:14,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:14,110 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:14:14,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:14,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:14:14,367 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T20:14:14,462 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T20:14:14,484 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T20:14:14,489 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:14:14,536 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2382 (auto-detected) 2024-11-26T20:14:14,538 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-26T20:14:14,576 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34091 2024-11-26T20:14:14,613 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34091 connecting to ZooKeeper ensemble=127.0.0.1:50191 2024-11-26T20:14:14,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:340910x0, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:14:14,737 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34091-0x10178c82cdf0000 connected 2024-11-26T20:14:14,927 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:14,937 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:14,985 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:14:14,991 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d, hbase.cluster.distributed=false 2024-11-26T20:14:15,031 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:14:15,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34091 2024-11-26T20:14:15,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34091 2024-11-26T20:14:15,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34091 2024-11-26T20:14:15,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34091 2024-11-26T20:14:15,047 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34091 2024-11-26T20:14:15,212 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/76919af20f29:0 server-side Connection retries=45 2024-11-26T20:14:15,215 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,215 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,215 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:14:15,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:14:15,220 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:14:15,224 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:14:15,231 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43267 2024-11-26T20:14:15,235 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43267 connecting to ZooKeeper ensemble=127.0.0.1:50191 2024-11-26T20:14:15,237 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:15,241 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:15,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:432670x0, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:14:15,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43267-0x10178c82cdf0001 connected 2024-11-26T20:14:15,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:14:15,286 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:14:15,315 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:14:15,336 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T20:14:15,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:14:15,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43267 2024-11-26T20:14:15,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43267 2024-11-26T20:14:15,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43267 2024-11-26T20:14:15,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43267 2024-11-26T20:14:15,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43267 2024-11-26T20:14:15,405 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/76919af20f29:0 server-side Connection retries=45 2024-11-26T20:14:15,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,406 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:14:15,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:14:15,407 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:14:15,407 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:14:15,413 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44765 2024-11-26T20:14:15,415 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44765 connecting to ZooKeeper ensemble=127.0.0.1:50191 2024-11-26T20:14:15,416 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:15,420 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:15,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447650x0, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:14:15,472 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44765-0x10178c82cdf0002 connected 2024-11-26T20:14:15,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:14:15,475 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:14:15,484 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:14:15,488 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T20:14:15,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:14:15,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44765 2024-11-26T20:14:15,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44765 2024-11-26T20:14:15,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44765 2024-11-26T20:14:15,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44765 2024-11-26T20:14:15,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44765 2024-11-26T20:14:15,557 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/76919af20f29:0 server-side Connection retries=45 2024-11-26T20:14:15,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,558 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T20:14:15,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T20:14:15,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T20:14:15,559 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T20:14:15,559 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T20:14:15,565 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38853 2024-11-26T20:14:15,572 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38853 connecting to ZooKeeper ensemble=127.0.0.1:50191 2024-11-26T20:14:15,574 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:15,581 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:15,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388530x0, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T20:14:15,632 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388530x0, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:14:15,633 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T20:14:15,634 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38853-0x10178c82cdf0003 connected 2024-11-26T20:14:15,634 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T20:14:15,636 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T20:14:15,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T20:14:15,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38853 2024-11-26T20:14:15,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38853 2024-11-26T20:14:15,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38853 2024-11-26T20:14:15,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38853 2024-11-26T20:14:15,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38853 2024-11-26T20:14:15,676 DEBUG [M:0;76919af20f29:34091 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;76919af20f29:34091 2024-11-26T20:14:15,679 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/76919af20f29,34091,1732652053862 2024-11-26T20:14:15,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:14:15,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:14:15,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:14:15,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:14:15,765 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/76919af20f29,34091,1732652053862 2024-11-26T20:14:15,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T20:14:15,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:15,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T20:14:15,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:15,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:15,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T20:14:15,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:15,847 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T20:14:15,853 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/76919af20f29,34091,1732652053862 from backup master directory 2024-11-26T20:14:15,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:14:15,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:14:15,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/76919af20f29,34091,1732652053862 2024-11-26T20:14:15,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:14:15,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T20:14:15,907 WARN [master/76919af20f29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:14:15,907 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=76919af20f29,34091,1732652053862 2024-11-26T20:14:15,910 INFO [master/76919af20f29:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T20:14:15,916 INFO [master/76919af20f29:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T20:14:16,038 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/hbase.id] with ID: 4c394a07-2298-4bb5-b984-38f62112b26d 2024-11-26T20:14:16,038 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/.tmp/hbase.id 2024-11-26T20:14:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741826_1002 (size=42) 2024-11-26T20:14:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741826_1002 (size=42) 2024-11-26T20:14:16,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741826_1002 (size=42) 2024-11-26T20:14:16,550 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/.tmp/hbase.id]:[hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/hbase.id] 2024-11-26T20:14:16,621 INFO [master/76919af20f29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T20:14:16,628 INFO [master/76919af20f29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T20:14:16,663 INFO [master/76919af20f29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 32ms. 2024-11-26T20:14:16,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:16,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:16,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:16,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:16,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741827_1003 (size=196) 2024-11-26T20:14:16,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741827_1003 (size=196) 2024-11-26T20:14:16,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741827_1003 (size=196) 2024-11-26T20:14:16,797 INFO [master/76919af20f29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:14:16,799 INFO [master/76919af20f29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T20:14:16,816 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:14:16,820 INFO [master/76919af20f29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:14:16,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741828_1004 (size=1189) 2024-11-26T20:14:16,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741828_1004 (size=1189) 2024-11-26T20:14:16,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741828_1004 (size=1189) 2024-11-26T20:14:16,892 INFO [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store 2024-11-26T20:14:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741829_1005 (size=34) 2024-11-26T20:14:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741829_1005 (size=34) 2024-11-26T20:14:16,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741829_1005 (size=34) 2024-11-26T20:14:16,966 INFO [master/76919af20f29:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T20:14:16,971 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:16,973 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T20:14:16,973 INFO [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:14:16,973 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:14:16,979 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-26T20:14:16,979 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:14:16,979 INFO [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:14:16,981 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732652056972Disabling compacts and flushes for region at 1732652056973 (+1 ms)Disabling writes for close at 1732652056979 (+6 ms)Writing region close event to WAL at 1732652056979Closed at 1732652056979 2024-11-26T20:14:16,985 WARN [master/76919af20f29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/.initializing 2024-11-26T20:14:16,986 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/WALs/76919af20f29,34091,1732652053862 2024-11-26T20:14:17,007 INFO [master/76919af20f29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:14:17,033 INFO [master/76919af20f29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=76919af20f29%2C34091%2C1732652053862, suffix=, logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/WALs/76919af20f29,34091,1732652053862, archiveDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/oldWALs, maxLogs=10 2024-11-26T20:14:17,060 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/WALs/76919af20f29,34091,1732652053862/76919af20f29%2C34091%2C1732652053862.1732652057039, exclude list is [], retry=0 2024-11-26T20:14:17,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK] 2024-11-26T20:14:17,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK] 2024-11-26T20:14:17,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK] 2024-11-26T20:14:17,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-26T20:14:17,177 INFO [master/76919af20f29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/WALs/76919af20f29,34091,1732652053862/76919af20f29%2C34091%2C1732652053862.1732652057039 2024-11-26T20:14:17,180 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33291:33291),(127.0.0.1/127.0.0.1:46547:46547),(127.0.0.1/127.0.0.1:37737:37737)] 2024-11-26T20:14:17,180 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:14:17,181 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:17,186 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,188 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,250 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,308 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T20:14:17,317 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:17,329 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:17,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,342 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T20:14:17,343 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:17,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:17,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,353 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T20:14:17,354 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:17,363 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:17,363 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,377 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T20:14:17,378 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:17,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:17,381 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,389 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,392 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,408 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,410 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,414 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:14:17,426 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T20:14:17,457 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:17,460 INFO [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66242992, jitterRate=-0.012902498245239258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:14:17,472 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732652057207Initializing all the Stores at 1732652057213 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652057214 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652057215 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652057215Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652057215Cleaning up temporary data from old regions at 1732652057410 (+195 ms)Region opened successfully at 1732652057471 (+61 ms) 2024-11-26T20:14:17,476 INFO [master/76919af20f29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T20:14:17,556 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388e82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=76919af20f29/172.17.0.3:0 2024-11-26T20:14:17,637 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T20:14:17,658 INFO [master/76919af20f29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:17,660 INFO [master/76919af20f29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:17,666 INFO [master/76919af20f29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:17,671 INFO [master/76919af20f29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 5 msec 2024-11-26T20:14:17,682 INFO [master/76919af20f29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T20:14:17,682 INFO [master/76919af20f29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T20:14:17,770 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T20:14:17,812 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T20:14:18,018 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T20:14:18,022 INFO [master/76919af20f29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T20:14:18,024 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T20:14:18,053 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T20:14:18,057 INFO [master/76919af20f29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T20:14:18,063 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T20:14:18,075 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T20:14:18,077 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T20:14:18,088 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T20:14:18,118 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T20:14:18,130 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T20:14:18,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:14:18,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:14:18,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:14:18,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T20:14:18,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,146 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=76919af20f29,34091,1732652053862, sessionid=0x10178c82cdf0000, setting cluster-up flag (Was=false) 2024-11-26T20:14:18,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,214 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T20:14:18,217 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=76919af20f29,34091,1732652053862 2024-11-26T20:14:18,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:18,278 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T20:14:18,281 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=76919af20f29,34091,1732652053862 2024-11-26T20:14:18,293 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T20:14:18,387 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(746): ClusterId : 4c394a07-2298-4bb5-b984-38f62112b26d 2024-11-26T20:14:18,387 INFO [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(746): ClusterId : 4c394a07-2298-4bb5-b984-38f62112b26d 2024-11-26T20:14:18,388 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(746): ClusterId : 4c394a07-2298-4bb5-b984-38f62112b26d 2024-11-26T20:14:18,392 DEBUG [RS:0;76919af20f29:43267 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:14:18,392 DEBUG [RS:1;76919af20f29:44765 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:14:18,394 DEBUG [RS:2;76919af20f29:38853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T20:14:18,457 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:18,477 INFO [master/76919af20f29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T20:14:18,487 INFO [master/76919af20f29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T20:14:18,499 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 76919af20f29,34091,1732652053862 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T20:14:18,506 DEBUG [RS:1;76919af20f29:44765 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:14:18,506 DEBUG [RS:1;76919af20f29:44765 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:14:18,512 DEBUG [RS:0;76919af20f29:43267 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:14:18,512 DEBUG [RS:0;76919af20f29:43267 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:14:18,521 DEBUG [RS:2;76919af20f29:38853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T20:14:18,521 DEBUG [RS:2;76919af20f29:38853 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T20:14:18,544 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/76919af20f29:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:14:18,544 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/76919af20f29:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:14:18,544 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/76919af20f29:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:14:18,545 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/76919af20f29:0, corePoolSize=5, maxPoolSize=5 2024-11-26T20:14:18,545 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/76919af20f29:0, corePoolSize=10, maxPoolSize=10 2024-11-26T20:14:18,545 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:18,545 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/76919af20f29:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:14:18,545 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:18,545 DEBUG [RS:0;76919af20f29:43267 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:14:18,546 DEBUG [RS:0;76919af20f29:43267 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eccef73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=76919af20f29/172.17.0.3:0 2024-11-26T20:14:18,552 DEBUG [RS:1;76919af20f29:44765 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:14:18,553 DEBUG [RS:1;76919af20f29:44765 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c9ef45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=76919af20f29/172.17.0.3:0 2024-11-26T20:14:18,558 INFO [master/76919af20f29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652088558 2024-11-26T20:14:18,560 DEBUG [RS:2;76919af20f29:38853 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T20:14:18,561 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T20:14:18,561 DEBUG [RS:2;76919af20f29:38853 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc4cfb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=76919af20f29/172.17.0.3:0 2024-11-26T20:14:18,562 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T20:14:18,567 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T20:14:18,571 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T20:14:18,571 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T20:14:18,572 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T20:14:18,581 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:18,582 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T20:14:18,586 DEBUG [RS:0;76919af20f29:43267 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;76919af20f29:43267 2024-11-26T20:14:18,590 INFO [RS:0;76919af20f29:43267 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:14:18,588 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:18,591 INFO [RS:0;76919af20f29:43267 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:14:18,591 DEBUG [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:14:18,594 DEBUG [RS:1;76919af20f29:44765 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;76919af20f29:44765 2024-11-26T20:14:18,594 INFO [RS:1;76919af20f29:44765 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:14:18,594 INFO [RS:1;76919af20f29:44765 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:14:18,594 DEBUG [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:14:18,600 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T20:14:18,601 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:18,602 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T20:14:18,602 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T20:14:18,602 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(2659): reportForDuty to master=76919af20f29,34091,1732652053862 with port=43267, startcode=1732652055155 2024-11-26T20:14:18,605 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(2659): reportForDuty to master=76919af20f29,34091,1732652053862 with port=44765, startcode=1732652055399 2024-11-26T20:14:18,602 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T20:14:18,610 DEBUG [RS:2;76919af20f29:38853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;76919af20f29:38853 2024-11-26T20:14:18,610 INFO [RS:2;76919af20f29:38853 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T20:14:18,610 INFO [RS:2;76919af20f29:38853 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T20:14:18,610 DEBUG [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T20:14:18,614 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T20:14:18,615 INFO [master/76919af20f29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T20:14:18,619 INFO [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(2659): reportForDuty to master=76919af20f29,34091,1732652053862 with port=38853, startcode=1732652055556 2024-11-26T20:14:18,641 DEBUG [RS:0;76919af20f29:43267 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:14:18,644 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/76919af20f29:0:becomeActiveMaster-HFileCleaner.large.0-1732652058617,5,FailOnTimeoutGroup] 2024-11-26T20:14:18,644 DEBUG [RS:2;76919af20f29:38853 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:14:18,653 DEBUG [RS:1;76919af20f29:44765 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T20:14:18,660 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/76919af20f29:0:becomeActiveMaster-HFileCleaner.small.0-1732652058648,5,FailOnTimeoutGroup] 2024-11-26T20:14:18,661 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:18,661 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T20:14:18,669 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:18,669 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:18,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741831_1007 (size=1321) 2024-11-26T20:14:18,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741831_1007 (size=1321) 2024-11-26T20:14:18,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741831_1007 (size=1321) 2024-11-26T20:14:18,798 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40955, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:14:18,798 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59641, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:14:18,798 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T20:14:18,799 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:18,801 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48611, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T20:14:18,821 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 76919af20f29,38853,1732652055556 2024-11-26T20:14:18,825 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] master.ServerManager(517): Registering regionserver=76919af20f29,38853,1732652055556 2024-11-26T20:14:18,862 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 76919af20f29,43267,1732652055155 2024-11-26T20:14:18,863 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] master.ServerManager(517): Registering regionserver=76919af20f29,43267,1732652055155 2024-11-26T20:14:18,871 DEBUG [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:18,871 DEBUG [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46543 2024-11-26T20:14:18,872 DEBUG [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:14:18,893 DEBUG [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:18,893 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 76919af20f29,44765,1732652055399 2024-11-26T20:14:18,893 DEBUG [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46543 2024-11-26T20:14:18,893 DEBUG [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:14:18,893 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] master.ServerManager(517): Registering regionserver=76919af20f29,44765,1732652055399 2024-11-26T20:14:18,902 DEBUG [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:18,902 DEBUG [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46543 2024-11-26T20:14:18,902 DEBUG [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T20:14:18,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741832_1008 (size=32) 2024-11-26T20:14:18,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741832_1008 (size=32) 2024-11-26T20:14:18,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:14:18,949 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:18,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741832_1008 (size=32) 2024-11-26T20:14:18,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T20:14:18,985 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:14:18,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:18,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:18,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T20:14:19,006 DEBUG [RS:0;76919af20f29:43267 {}] zookeeper.ZKUtil(111): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/76919af20f29,43267,1732652055155 2024-11-26T20:14:19,006 DEBUG [RS:2;76919af20f29:38853 {}] zookeeper.ZKUtil(111): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/76919af20f29,38853,1732652055556 2024-11-26T20:14:19,006 WARN [RS:0;76919af20f29:43267 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:14:19,006 WARN [RS:2;76919af20f29:38853 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:14:19,006 INFO [RS:2;76919af20f29:38853 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:14:19,006 INFO [RS:0;76919af20f29:43267 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:14:19,006 DEBUG [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556 2024-11-26T20:14:19,006 DEBUG [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,43267,1732652055155 2024-11-26T20:14:19,009 DEBUG [RS:1;76919af20f29:44765 {}] zookeeper.ZKUtil(111): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/76919af20f29,44765,1732652055399 2024-11-26T20:14:19,009 WARN [RS:1;76919af20f29:44765 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T20:14:19,009 INFO [RS:1;76919af20f29:44765 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:14:19,009 DEBUG [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,44765,1732652055399 2024-11-26T20:14:19,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:14:19,011 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:19,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [76919af20f29,43267,1732652055155] 2024-11-26T20:14:19,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [76919af20f29,38853,1732652055556] 2024-11-26T20:14:19,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [76919af20f29,44765,1732652055399] 2024-11-26T20:14:19,024 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:19,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T20:14:19,044 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:14:19,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:19,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:19,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T20:14:19,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:14:19,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:19,069 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:19,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:14:19,074 INFO [RS:2;76919af20f29:38853 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:14:19,076 INFO [RS:0;76919af20f29:43267 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:14:19,081 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740 2024-11-26T20:14:19,084 INFO [RS:1;76919af20f29:44765 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T20:14:19,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740 2024-11-26T20:14:19,123 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:14:19,123 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:14:19,131 INFO [RS:2;76919af20f29:38853 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:14:19,135 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:14:19,140 INFO [RS:1;76919af20f29:44765 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:14:19,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:14:19,152 INFO [RS:0;76919af20f29:43267 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T20:14:19,155 INFO [RS:2;76919af20f29:38853 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:14:19,155 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,156 INFO [RS:0;76919af20f29:43267 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:14:19,157 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,164 INFO [RS:1;76919af20f29:44765 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T20:14:19,165 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,165 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:14:19,168 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:14:19,169 INFO [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T20:14:19,173 INFO [RS:1;76919af20f29:44765 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:14:19,175 INFO [RS:0;76919af20f29:43267 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:14:19,176 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,176 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,177 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,177 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,177 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,177 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,177 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,177 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:14:19,178 INFO [RS:2;76919af20f29:38853 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T20:14:19,178 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,179 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,179 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,179 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:19,179 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,179 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,179 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,180 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,180 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,180 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:14:19,180 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,180 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0, corePoolSize=2, maxPoolSize=2 2024-11-26T20:14:19,181 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,181 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/76919af20f29:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:14:19,182 DEBUG [RS:2;76919af20f29:38853 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/76919af20f29:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:14:19,182 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,182 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,182 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,182 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/76919af20f29:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:14:19,182 DEBUG [RS:1;76919af20f29:44765 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/76919af20f29:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:14:19,178 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,182 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,183 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,183 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,183 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,183 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/76919af20f29:0, corePoolSize=1, maxPoolSize=1 2024-11-26T20:14:19,183 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74144842, jitterRate=0.10484424233436584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:14:19,183 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/76919af20f29:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:14:19,184 DEBUG [RS:0;76919af20f29:43267 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/76919af20f29:0, corePoolSize=3, maxPoolSize=3 2024-11-26T20:14:19,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732652058949Initializing all the Stores at 1732652058957 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652058960 (+3 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652058964 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652058965 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652058965Cleaning up temporary data from old regions at 1732652059123 (+158 ms)Region opened successfully at 1732652059185 (+62 ms) 2024-11-26T20:14:19,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T20:14:19,186 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T20:14:19,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T20:14:19,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T20:14:19,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T20:14:19,193 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,193 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,194 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,194 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,194 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,194 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,38853,1732652055556-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:14:19,213 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T20:14:19,214 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732652059186Disabling compacts and flushes for region at 1732652059186Disabling writes for close at 1732652059186Writing region close event to WAL at 1732652059213 (+27 ms)Closed at 1732652059213 2024-11-26T20:14:19,231 INFO [RS:2;76919af20f29:38853 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:14:19,235 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,38853,1732652055556-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,236 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,236 INFO [RS:2;76919af20f29:38853 {}] regionserver.Replication(171): 76919af20f29,38853,1732652055556 started 2024-11-26T20:14:19,262 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,262 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,262 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,263 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,263 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,263 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:19,263 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,43267,1732652055155-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:14:19,263 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T20:14:19,272 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,272 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,273 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,273 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,273 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,273 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,44765,1732652055399-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:14:19,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T20:14:19,286 INFO [RS:2;76919af20f29:38853 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,289 INFO [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(1482): Serving as 76919af20f29,38853,1732652055556, RpcServer on 76919af20f29/172.17.0.3:38853, sessionid=0x10178c82cdf0003 2024-11-26T20:14:19,290 DEBUG [RS:2;76919af20f29:38853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:14:19,291 DEBUG [RS:2;76919af20f29:38853 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 76919af20f29,38853,1732652055556 2024-11-26T20:14:19,291 DEBUG [RS:2;76919af20f29:38853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '76919af20f29,38853,1732652055556' 2024-11-26T20:14:19,298 DEBUG [RS:2;76919af20f29:38853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:14:19,302 INFO [RS:1;76919af20f29:44765 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:14:19,302 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,44765,1732652055399-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,304 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,306 INFO [RS:1;76919af20f29:44765 {}] regionserver.Replication(171): 76919af20f29,44765,1732652055399 started 2024-11-26T20:14:19,306 DEBUG [RS:2;76919af20f29:38853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:14:19,309 DEBUG [RS:2;76919af20f29:38853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:14:19,309 DEBUG [RS:2;76919af20f29:38853 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:14:19,309 DEBUG [RS:2;76919af20f29:38853 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 76919af20f29,38853,1732652055556 2024-11-26T20:14:19,309 DEBUG [RS:2;76919af20f29:38853 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '76919af20f29,38853,1732652055556' 2024-11-26T20:14:19,310 DEBUG [RS:2;76919af20f29:38853 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:14:19,311 INFO [RS:0;76919af20f29:43267 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T20:14:19,311 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,43267,1732652055155-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,312 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,312 INFO [RS:0;76919af20f29:43267 {}] regionserver.Replication(171): 76919af20f29,43267,1732652055155 started 2024-11-26T20:14:19,321 DEBUG [RS:2;76919af20f29:38853 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:14:19,322 DEBUG [RS:2;76919af20f29:38853 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:14:19,322 INFO [RS:2;76919af20f29:38853 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:14:19,322 INFO [RS:2;76919af20f29:38853 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:14:19,325 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:19,331 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,331 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1482): Serving as 76919af20f29,44765,1732652055399, RpcServer on 76919af20f29/172.17.0.3:44765, sessionid=0x10178c82cdf0002 2024-11-26T20:14:19,332 DEBUG [RS:1;76919af20f29:44765 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:14:19,332 DEBUG [RS:1;76919af20f29:44765 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 76919af20f29,44765,1732652055399 2024-11-26T20:14:19,332 DEBUG [RS:1;76919af20f29:44765 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '76919af20f29,44765,1732652055399' 2024-11-26T20:14:19,332 DEBUG [RS:1;76919af20f29:44765 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:14:19,333 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T20:14:19,335 DEBUG [RS:1;76919af20f29:44765 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:14:19,339 DEBUG [RS:1;76919af20f29:44765 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:14:19,339 DEBUG [RS:1;76919af20f29:44765 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:14:19,339 DEBUG [RS:1;76919af20f29:44765 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 76919af20f29,44765,1732652055399 2024-11-26T20:14:19,339 DEBUG [RS:1;76919af20f29:44765 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '76919af20f29,44765,1732652055399' 2024-11-26T20:14:19,339 DEBUG [RS:1;76919af20f29:44765 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:14:19,340 DEBUG [RS:1;76919af20f29:44765 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:14:19,342 DEBUG [RS:1;76919af20f29:44765 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:14:19,342 INFO [RS:1;76919af20f29:44765 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:14:19,343 INFO [RS:1;76919af20f29:44765 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:14:19,351 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:19,351 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1482): Serving as 76919af20f29,43267,1732652055155, RpcServer on 76919af20f29/172.17.0.3:43267, sessionid=0x10178c82cdf0001 2024-11-26T20:14:19,351 DEBUG [RS:0;76919af20f29:43267 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T20:14:19,352 DEBUG [RS:0;76919af20f29:43267 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 76919af20f29,43267,1732652055155 2024-11-26T20:14:19,352 DEBUG [RS:0;76919af20f29:43267 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '76919af20f29,43267,1732652055155' 2024-11-26T20:14:19,352 DEBUG [RS:0;76919af20f29:43267 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T20:14:19,353 DEBUG [RS:0;76919af20f29:43267 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T20:14:19,355 DEBUG [RS:0;76919af20f29:43267 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T20:14:19,355 DEBUG [RS:0;76919af20f29:43267 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T20:14:19,355 DEBUG [RS:0;76919af20f29:43267 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 76919af20f29,43267,1732652055155 2024-11-26T20:14:19,355 DEBUG [RS:0;76919af20f29:43267 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '76919af20f29,43267,1732652055155' 2024-11-26T20:14:19,355 DEBUG [RS:0;76919af20f29:43267 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T20:14:19,358 DEBUG [RS:0;76919af20f29:43267 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T20:14:19,361 DEBUG [RS:0;76919af20f29:43267 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T20:14:19,361 INFO [RS:0;76919af20f29:43267 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T20:14:19,362 INFO [RS:0;76919af20f29:43267 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T20:14:19,431 INFO [RS:2;76919af20f29:38853 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:14:19,437 INFO [RS:2;76919af20f29:38853 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=76919af20f29%2C38853%2C1732652055556, suffix=, logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556, archiveDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs, maxLogs=32 2024-11-26T20:14:19,444 INFO [RS:1;76919af20f29:44765 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:14:19,450 INFO [RS:1;76919af20f29:44765 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=76919af20f29%2C44765%2C1732652055399, suffix=, logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,44765,1732652055399, archiveDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs, maxLogs=32 2024-11-26T20:14:19,463 INFO [RS:0;76919af20f29:43267 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-26T20:14:19,465 DEBUG [RS:2;76919af20f29:38853 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556/76919af20f29%2C38853%2C1732652055556.1732652059440, exclude list is [], retry=0 2024-11-26T20:14:19,479 INFO [RS:0;76919af20f29:43267 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=76919af20f29%2C43267%2C1732652055155, suffix=, logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,43267,1732652055155, archiveDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs, maxLogs=32 2024-11-26T20:14:19,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK] 2024-11-26T20:14:19,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK] 2024-11-26T20:14:19,484 WARN [76919af20f29:34091 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T20:14:19,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK] 2024-11-26T20:14:19,484 DEBUG [RS:1;76919af20f29:44765 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,44765,1732652055399/76919af20f29%2C44765%2C1732652055399.1732652059453, exclude list is [], retry=0 2024-11-26T20:14:19,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK] 2024-11-26T20:14:19,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK] 2024-11-26T20:14:19,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK] 2024-11-26T20:14:19,507 DEBUG [RS:0;76919af20f29:43267 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,43267,1732652055155/76919af20f29%2C43267%2C1732652055155.1732652059481, exclude list is [], retry=0 2024-11-26T20:14:19,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK] 2024-11-26T20:14:19,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK] 2024-11-26T20:14:19,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK] 2024-11-26T20:14:19,537 INFO [RS:2;76919af20f29:38853 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556/76919af20f29%2C38853%2C1732652055556.1732652059440 2024-11-26T20:14:19,541 INFO [RS:1;76919af20f29:44765 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,44765,1732652055399/76919af20f29%2C44765%2C1732652055399.1732652059453 2024-11-26T20:14:19,542 DEBUG [RS:2;76919af20f29:38853 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33291:33291),(127.0.0.1/127.0.0.1:46547:46547),(127.0.0.1/127.0.0.1:37737:37737)] 2024-11-26T20:14:19,545 DEBUG [RS:1;76919af20f29:44765 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33291:33291),(127.0.0.1/127.0.0.1:37737:37737),(127.0.0.1/127.0.0.1:46547:46547)] 2024-11-26T20:14:19,561 INFO [RS:0;76919af20f29:43267 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,43267,1732652055155/76919af20f29%2C43267%2C1732652055155.1732652059481 2024-11-26T20:14:19,568 DEBUG [RS:0;76919af20f29:43267 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33291:33291),(127.0.0.1/127.0.0.1:37737:37737),(127.0.0.1/127.0.0.1:46547:46547)] 2024-11-26T20:14:19,739 DEBUG [76919af20f29:34091 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-26T20:14:19,753 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(204): Hosts are {76919af20f29=0} racks are {/default-rack=0} 2024-11-26T20:14:19,771 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T20:14:19,771 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T20:14:19,771 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T20:14:19,771 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:14:19,772 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:14:19,772 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T20:14:19,772 INFO [76919af20f29:34091 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:14:19,772 INFO [76919af20f29:34091 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:14:19,772 INFO [76919af20f29:34091 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T20:14:19,772 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T20:14:19,784 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:19,800 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,38853,1732652055556, state=OPENING 2024-11-26T20:14:19,856 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T20:14:19,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:19,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:19,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:19,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:19,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:19,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:19,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:19,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:19,872 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:19,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:20,080 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T20:14:20,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46747, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T20:14:20,115 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T20:14:20,117 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:14:20,118 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T20:14:20,141 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=76919af20f29%2C38853%2C1732652055556.meta, suffix=.meta, logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556, archiveDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs, maxLogs=32 2024-11-26T20:14:20,171 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, exclude list is [], retry=0 2024-11-26T20:14:20,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK] 2024-11-26T20:14:20,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK] 2024-11-26T20:14:20,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK] 2024-11-26T20:14:20,233 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta 2024-11-26T20:14:20,240 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33291:33291),(127.0.0.1/127.0.0.1:46547:46547),(127.0.0.1/127.0.0.1:37737:37737)] 2024-11-26T20:14:20,241 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:14:20,244 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T20:14:20,248 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T20:14:20,257 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T20:14:20,266 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T20:14:20,268 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:20,268 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T20:14:20,268 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T20:14:20,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T20:14:20,282 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:14:20,282 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:20,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:20,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T20:14:20,287 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:14:20,287 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:20,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:20,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T20:14:20,298 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:14:20,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:20,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:20,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T20:14:20,302 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:14:20,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:20,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:20,304 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:14:20,307 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740 2024-11-26T20:14:20,319 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740 2024-11-26T20:14:20,331 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:14:20,331 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:14:20,333 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:14:20,339 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:14:20,345 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63639030, jitterRate=-0.05170455574989319}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:14:20,346 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T20:14:20,348 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732652060269Writing region info on filesystem at 1732652060270 (+1 ms)Initializing all the Stores at 1732652060278 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652060278Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652060279 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652060279Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652060279Cleaning up temporary data from old regions at 1732652060331 (+52 ms)Running coprocessor post-open hooks at 1732652060346 (+15 ms)Region opened successfully at 1732652060348 (+2 ms) 2024-11-26T20:14:20,362 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732652060066 2024-11-26T20:14:20,400 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T20:14:20,401 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T20:14:20,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:20,425 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,38853,1732652055556, state=OPEN 2024-11-26T20:14:20,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:20,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:20,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:20,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:20,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:20,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:20,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:20,653 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:20,654 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=76919af20f29,38853,1732652055556 2024-11-26T20:14:20,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T20:14:20,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,38853,1732652055556 in 779 msec 2024-11-26T20:14:20,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T20:14:20,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.3940 sec 2024-11-26T20:14:20,679 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:20,679 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T20:14:20,709 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:14:20,712 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=76919af20f29,38853,1732652055556, seqNum=-1] 2024-11-26T20:14:20,743 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:14:20,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40865, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:14:20,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.4460 sec 2024-11-26T20:14:20,811 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732652060811, completionTime=-1 2024-11-26T20:14:20,816 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-26T20:14:20,816 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:20,867 INFO [master/76919af20f29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-26T20:14:20,868 INFO [master/76919af20f29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652120868 2024-11-26T20:14:20,868 INFO [master/76919af20f29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652180868 2024-11-26T20:14:20,872 INFO [master/76919af20f29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 53 msec 2024-11-26T20:14:20,883 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:20,905 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,34091,1732652053862-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:20,906 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,34091,1732652053862-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:20,906 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,34091,1732652053862-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:20,908 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-76919af20f29:34091, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:20,915 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:20,917 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:20,922 DEBUG [master/76919af20f29:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T20:14:20,957 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 5.050sec 2024-11-26T20:14:20,960 INFO [master/76919af20f29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T20:14:20,962 INFO [master/76919af20f29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T20:14:20,964 INFO [master/76919af20f29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T20:14:20,965 INFO [master/76919af20f29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T20:14:20,965 INFO [master/76919af20f29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T20:14:20,966 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,34091,1732652053862-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T20:14:20,966 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,34091,1732652053862-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T20:14:20,986 DEBUG [master/76919af20f29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T20:14:20,987 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T20:14:20,988 INFO [master/76919af20f29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=76919af20f29,34091,1732652053862-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T20:14:21,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b8910, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:14:21,026 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T20:14:21,026 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T20:14:21,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 76919af20f29,34091,-1 for getting cluster id 2024-11-26T20:14:21,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T20:14:21,068 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4c394a07-2298-4bb5-b984-38f62112b26d' 2024-11-26T20:14:21,071 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T20:14:21,072 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4c394a07-2298-4bb5-b984-38f62112b26d" 2024-11-26T20:14:21,074 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132e81fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:14:21,074 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [76919af20f29,34091,-1] 2024-11-26T20:14:21,078 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T20:14:21,084 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:21,087 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T20:14:21,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55599eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T20:14:21,091 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:14:21,099 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=76919af20f29,38853,1732652055556, seqNum=-1] 2024-11-26T20:14:21,099 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:14:21,102 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:14:21,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=76919af20f29,34091,1732652053862 2024-11-26T20:14:21,132 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T20:14:21,138 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 76919af20f29,34091,1732652053862 2024-11-26T20:14:21,169 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65c4a3e1 2024-11-26T20:14:21,171 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T20:14:21,175 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T20:14:21,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T20:14:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:21,203 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T20:14:21,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-26T20:14:21,211 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:21,218 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T20:14:21,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741837_1013 (size=358) 2024-11-26T20:14:21,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741837_1013 (size=358) 2024-11-26T20:14:21,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741837_1013 (size=358) 2024-11-26T20:14:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:21,345 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5070f0e2d0547381b8d60e94c76cdaff, NAME => 'test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,362 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4b8a0db8a2157574a65d6d70ba9f15e1, NAME => 'test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,381 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a7108f1289f21074490c53bbef06d630, NAME => 'test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741838_1014 (size=42) 2024-11-26T20:14:21,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741838_1014 (size=42) 2024-11-26T20:14:21,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741838_1014 (size=42) 2024-11-26T20:14:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:21,542 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,542 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5070f0e2d0547381b8d60e94c76cdaff, disabling compactions & flushes 2024-11-26T20:14:21,542 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:21,542 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:21,543 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. after waiting 0 ms 2024-11-26T20:14:21,543 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:21,543 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:21,543 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5070f0e2d0547381b8d60e94c76cdaff: Waiting for close lock at 1732652061542Disabling compacts and flushes for region at 1732652061542Disabling writes for close at 1732652061543 (+1 ms)Writing region close event to WAL at 1732652061543Closed at 1732652061543 2024-11-26T20:14:21,548 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e22e09e90e1a05e430419c43da9c80c, NAME => 'test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741839_1015 (size=45) 2024-11-26T20:14:21,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741839_1015 (size=45) 2024-11-26T20:14:21,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741839_1015 (size=45) 2024-11-26T20:14:21,570 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,570 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4b8a0db8a2157574a65d6d70ba9f15e1, disabling compactions & flushes 2024-11-26T20:14:21,570 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:21,570 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:21,570 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. after waiting 0 ms 2024-11-26T20:14:21,570 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:21,570 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:21,570 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4b8a0db8a2157574a65d6d70ba9f15e1: Waiting for close lock at 1732652061570Disabling compacts and flushes for region at 1732652061570Disabling writes for close at 1732652061570Writing region close event to WAL at 1732652061570Closed at 1732652061570 2024-11-26T20:14:21,571 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e78554385cc62e1f4a220f96a9d9207d, NAME => 'test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741840_1016 (size=45) 2024-11-26T20:14:21,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741840_1016 (size=45) 2024-11-26T20:14:21,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741840_1016 (size=45) 2024-11-26T20:14:21,638 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,638 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing a7108f1289f21074490c53bbef06d630, disabling compactions & flushes 2024-11-26T20:14:21,638 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:21,638 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:21,638 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. after waiting 0 ms 2024-11-26T20:14:21,638 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:21,638 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:21,638 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for a7108f1289f21074490c53bbef06d630: Waiting for close lock at 1732652061638Disabling compacts and flushes for region at 1732652061638Disabling writes for close at 1732652061638Writing region close event to WAL at 1732652061638Closed at 1732652061638 2024-11-26T20:14:21,639 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5632c367c5d9113b557fd742bcb2b62f, NAME => 'test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741841_1017 (size=45) 2024-11-26T20:14:21,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741841_1017 (size=45) 2024-11-26T20:14:21,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741841_1017 (size=45) 2024-11-26T20:14:21,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741842_1018 (size=45) 2024-11-26T20:14:21,710 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,710 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1e22e09e90e1a05e430419c43da9c80c, disabling compactions & flushes 2024-11-26T20:14:21,710 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:21,710 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:21,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741842_1018 (size=45) 2024-11-26T20:14:21,710 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. after waiting 0 ms 2024-11-26T20:14:21,710 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:21,710 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:21,711 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e22e09e90e1a05e430419c43da9c80c: Waiting for close lock at 1732652061710Disabling compacts and flushes for region at 1732652061710Disabling writes for close at 1732652061710Writing region close event to WAL at 1732652061710Closed at 1732652061710 2024-11-26T20:14:21,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741842_1018 (size=45) 2024-11-26T20:14:21,713 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 34cb9834732a35daa158cb86b8b3b142, NAME => 'test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,716 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,716 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e78554385cc62e1f4a220f96a9d9207d, disabling compactions & flushes 2024-11-26T20:14:21,716 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:21,716 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:21,716 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. after waiting 0 ms 2024-11-26T20:14:21,716 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:21,716 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:21,717 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e78554385cc62e1f4a220f96a9d9207d: Waiting for close lock at 1732652061716Disabling compacts and flushes for region at 1732652061716Disabling writes for close at 1732652061716Writing region close event to WAL at 1732652061716Closed at 1732652061716 2024-11-26T20:14:21,718 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dde94c133aedae28d634a201ddb741f7, NAME => 'test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741843_1019 (size=45) 2024-11-26T20:14:21,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741843_1019 (size=45) 2024-11-26T20:14:21,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741843_1019 (size=45) 2024-11-26T20:14:21,767 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,767 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5632c367c5d9113b557fd742bcb2b62f, disabling compactions & flushes 2024-11-26T20:14:21,767 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:21,767 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:21,767 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. after waiting 0 ms 2024-11-26T20:14:21,767 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:21,767 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:21,768 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5632c367c5d9113b557fd742bcb2b62f: Waiting for close lock at 1732652061767Disabling compacts and flushes for region at 1732652061767Disabling writes for close at 1732652061767Writing region close event to WAL at 1732652061767Closed at 1732652061767 2024-11-26T20:14:21,768 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 87979a5e316fc73bab2f01285393a426, NAME => 'test,iii,1732652061178.87979a5e316fc73bab2f01285393a426.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741844_1020 (size=45) 2024-11-26T20:14:21,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741844_1020 (size=45) 2024-11-26T20:14:21,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741844_1020 (size=45) 2024-11-26T20:14:21,789 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,789 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 34cb9834732a35daa158cb86b8b3b142, disabling compactions & flushes 2024-11-26T20:14:21,789 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:21,789 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:21,789 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. after waiting 0 ms 2024-11-26T20:14:21,789 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:21,790 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:21,790 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 34cb9834732a35daa158cb86b8b3b142: Waiting for close lock at 1732652061789Disabling compacts and flushes for region at 1732652061789Disabling writes for close at 1732652061789Writing region close event to WAL at 1732652061789Closed at 1732652061790 (+1 ms) 2024-11-26T20:14:21,791 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2ca44b38774b13852ff96928138ba075, NAME => 'test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741845_1021 (size=45) 2024-11-26T20:14:21,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741845_1021 (size=45) 2024-11-26T20:14:21,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741845_1021 (size=45) 2024-11-26T20:14:21,795 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,795 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing dde94c133aedae28d634a201ddb741f7, disabling compactions & flushes 2024-11-26T20:14:21,795 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:21,795 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:21,796 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. after waiting 0 ms 2024-11-26T20:14:21,796 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:21,796 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:21,796 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for dde94c133aedae28d634a201ddb741f7: Waiting for close lock at 1732652061795Disabling compacts and flushes for region at 1732652061795Disabling writes for close at 1732652061796 (+1 ms)Writing region close event to WAL at 1732652061796Closed at 1732652061796 2024-11-26T20:14:21,801 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f5d1c8271bbf4e7a65c1cc98ef8dc026, NAME => 'test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:21,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741847_1023 (size=45) 2024-11-26T20:14:21,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741847_1023 (size=45) 2024-11-26T20:14:21,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741847_1023 (size=45) 2024-11-26T20:14:21,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f5d1c8271bbf4e7a65c1cc98ef8dc026, disabling compactions & flushes 2024-11-26T20:14:21,870 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:21,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:21,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. after waiting 0 ms 2024-11-26T20:14:21,870 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:21,870 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:21,871 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f5d1c8271bbf4e7a65c1cc98ef8dc026: Waiting for close lock at 1732652061870Disabling compacts and flushes for region at 1732652061870Disabling writes for close at 1732652061870Writing region close event to WAL at 1732652061870Closed at 1732652061870 2024-11-26T20:14:21,872 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c5249ac65133ea81926c57db86f9b90e, NAME => 'test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741846_1022 (size=45) 2024-11-26T20:14:21,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741846_1022 (size=45) 2024-11-26T20:14:21,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741846_1022 (size=45) 2024-11-26T20:14:21,887 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,iii,1732652061178.87979a5e316fc73bab2f01285393a426.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,887 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 87979a5e316fc73bab2f01285393a426, disabling compactions & flushes 2024-11-26T20:14:21,887 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:21,887 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:21,888 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. after waiting 0 ms 2024-11-26T20:14:21,888 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:21,888 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:21,888 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 87979a5e316fc73bab2f01285393a426: Waiting for close lock at 1732652061887Disabling compacts and flushes for region at 1732652061887Disabling writes for close at 1732652061888 (+1 ms)Writing region close event to WAL at 1732652061888Closed at 1732652061888 2024-11-26T20:14:21,889 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0a569d698819974eb6636a41f610467c, NAME => 'test,mmm,1732652061178.0a569d698819974eb6636a41f610467c.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741848_1024 (size=45) 2024-11-26T20:14:21,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741848_1024 (size=45) 2024-11-26T20:14:21,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741848_1024 (size=45) 2024-11-26T20:14:21,921 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2ca44b38774b13852ff96928138ba075, disabling compactions & flushes 2024-11-26T20:14:21,925 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:21,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:21,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. after waiting 0 ms 2024-11-26T20:14:21,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:21,925 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:21,925 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2ca44b38774b13852ff96928138ba075: Waiting for close lock at 1732652061924Disabling compacts and flushes for region at 1732652061924Disabling writes for close at 1732652061925 (+1 ms)Writing region close event to WAL at 1732652061925Closed at 1732652061925 2024-11-26T20:14:21,926 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 340a1869e06658f75d054f357d9a5c66, NAME => 'test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741849_1025 (size=45) 2024-11-26T20:14:21,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741849_1025 (size=45) 2024-11-26T20:14:21,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741849_1025 (size=45) 2024-11-26T20:14:21,975 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,975 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c5249ac65133ea81926c57db86f9b90e, disabling compactions & flushes 2024-11-26T20:14:21,975 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:21,976 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:21,976 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. after waiting 0 ms 2024-11-26T20:14:21,976 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:21,976 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:21,976 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c5249ac65133ea81926c57db86f9b90e: Waiting for close lock at 1732652061975Disabling compacts and flushes for region at 1732652061975Disabling writes for close at 1732652061976 (+1 ms)Writing region close event to WAL at 1732652061976Closed at 1732652061976 2024-11-26T20:14:21,977 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 78cd4b9200f83099c4095aeeffca9b21, NAME => 'test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741851_1027 (size=45) 2024-11-26T20:14:21,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741851_1027 (size=45) 2024-11-26T20:14:21,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741851_1027 (size=45) 2024-11-26T20:14:21,987 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:21,987 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 340a1869e06658f75d054f357d9a5c66, disabling compactions & flushes 2024-11-26T20:14:21,987 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:21,988 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:21,988 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. after waiting 0 ms 2024-11-26T20:14:21,988 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:21,988 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:21,988 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 340a1869e06658f75d054f357d9a5c66: Waiting for close lock at 1732652061987Disabling compacts and flushes for region at 1732652061987Disabling writes for close at 1732652061988 (+1 ms)Writing region close event to WAL at 1732652061988Closed at 1732652061988 2024-11-26T20:14:21,992 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1c0c8c5fd0d7b81fc270155b15f8b86, NAME => 'test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741850_1026 (size=45) 2024-11-26T20:14:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741850_1026 (size=45) 2024-11-26T20:14:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741850_1026 (size=45) 2024-11-26T20:14:22,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741852_1028 (size=45) 2024-11-26T20:14:22,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741852_1028 (size=45) 2024-11-26T20:14:22,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741852_1028 (size=45) 2024-11-26T20:14:22,062 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,063 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 78cd4b9200f83099c4095aeeffca9b21, disabling compactions & flushes 2024-11-26T20:14:22,063 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:22,063 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:22,063 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. after waiting 0 ms 2024-11-26T20:14:22,063 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:22,063 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:22,063 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 78cd4b9200f83099c4095aeeffca9b21: Waiting for close lock at 1732652062063Disabling compacts and flushes for region at 1732652062063Disabling writes for close at 1732652062063Writing region close event to WAL at 1732652062063Closed at 1732652062063 2024-11-26T20:14:22,064 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 726bb7d945255f0a9e4ebbb78c65b7d5, NAME => 'test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741853_1029 (size=45) 2024-11-26T20:14:22,094 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,095 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d1c0c8c5fd0d7b81fc270155b15f8b86, disabling compactions & flushes 2024-11-26T20:14:22,095 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:22,095 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:22,095 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. after waiting 0 ms 2024-11-26T20:14:22,095 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:22,095 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:22,095 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1c0c8c5fd0d7b81fc270155b15f8b86: Waiting for close lock at 1732652062095Disabling compacts and flushes for region at 1732652062095Disabling writes for close at 1732652062095Writing region close event to WAL at 1732652062095Closed at 1732652062095 2024-11-26T20:14:22,096 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ba6d284e3fe19dfa6e13c7648e255291, NAME => 'test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741853_1029 (size=45) 2024-11-26T20:14:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741853_1029 (size=45) 2024-11-26T20:14:22,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741854_1030 (size=45) 2024-11-26T20:14:22,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741854_1030 (size=45) 2024-11-26T20:14:22,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741854_1030 (size=45) 2024-11-26T20:14:22,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 726bb7d945255f0a9e4ebbb78c65b7d5, disabling compactions & flushes 2024-11-26T20:14:22,131 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:22,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:22,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. after waiting 0 ms 2024-11-26T20:14:22,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:22,131 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:22,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 726bb7d945255f0a9e4ebbb78c65b7d5: Waiting for close lock at 1732652062131Disabling compacts and flushes for region at 1732652062131Disabling writes for close at 1732652062131Writing region close event to WAL at 1732652062131Closed at 1732652062131 2024-11-26T20:14:22,132 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c156031df6143a0a099724d68db39599, NAME => 'test,sss,1732652061178.c156031df6143a0a099724d68db39599.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741855_1031 (size=45) 2024-11-26T20:14:22,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741855_1031 (size=45) 2024-11-26T20:14:22,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741855_1031 (size=45) 2024-11-26T20:14:22,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ba6d284e3fe19dfa6e13c7648e255291, disabling compactions & flushes 2024-11-26T20:14:22,175 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:22,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:22,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. after waiting 0 ms 2024-11-26T20:14:22,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:22,175 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:22,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ba6d284e3fe19dfa6e13c7648e255291: Waiting for close lock at 1732652062175Disabling compacts and flushes for region at 1732652062175Disabling writes for close at 1732652062175Writing region close event to WAL at 1732652062175Closed at 1732652062175 2024-11-26T20:14:22,177 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0c6f3b0d7be2eee5a98bcf417b7ba9e, NAME => 'test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741856_1032 (size=45) 2024-11-26T20:14:22,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741856_1032 (size=45) 2024-11-26T20:14:22,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741856_1032 (size=45) 2024-11-26T20:14:22,217 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732652061178.c156031df6143a0a099724d68db39599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,217 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c156031df6143a0a099724d68db39599, disabling compactions & flushes 2024-11-26T20:14:22,217 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:22,217 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:22,217 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732652061178.c156031df6143a0a099724d68db39599. after waiting 0 ms 2024-11-26T20:14:22,217 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:22,217 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:22,217 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c156031df6143a0a099724d68db39599: Waiting for close lock at 1732652062217Disabling compacts and flushes for region at 1732652062217Disabling writes for close at 1732652062217Writing region close event to WAL at 1732652062217Closed at 1732652062217 2024-11-26T20:14:22,218 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 39562e7939506cf96cc70158f24d8ea2, NAME => 'test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741857_1033 (size=45) 2024-11-26T20:14:22,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741857_1033 (size=45) 2024-11-26T20:14:22,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741857_1033 (size=45) 2024-11-26T20:14:22,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c0c6f3b0d7be2eee5a98bcf417b7ba9e, disabling compactions & flushes 2024-11-26T20:14:22,247 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:22,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:22,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. after waiting 0 ms 2024-11-26T20:14:22,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:22,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:22,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0c6f3b0d7be2eee5a98bcf417b7ba9e: Waiting for close lock at 1732652062247Disabling compacts and flushes for region at 1732652062247Disabling writes for close at 1732652062247Writing region close event to WAL at 1732652062248 (+1 ms)Closed at 1732652062248 2024-11-26T20:14:22,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 10a9d36c52dee7a92acb00483dfe6052, NAME => 'test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741858_1034 (size=45) 2024-11-26T20:14:22,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741858_1034 (size=45) 2024-11-26T20:14:22,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741858_1034 (size=45) 2024-11-26T20:14:22,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 39562e7939506cf96cc70158f24d8ea2, disabling compactions & flushes 2024-11-26T20:14:22,294 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:22,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:22,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. after waiting 0 ms 2024-11-26T20:14:22,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:22,294 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:22,294 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 39562e7939506cf96cc70158f24d8ea2: Waiting for close lock at 1732652062294Disabling compacts and flushes for region at 1732652062294Disabling writes for close at 1732652062294Writing region close event to WAL at 1732652062294Closed at 1732652062294 2024-11-26T20:14:22,295 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0880a54a28990b0c067cdb9447a1593b, NAME => 'test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741859_1035 (size=45) 2024-11-26T20:14:22,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741859_1035 (size=45) 2024-11-26T20:14:22,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741859_1035 (size=45) 2024-11-26T20:14:22,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 10a9d36c52dee7a92acb00483dfe6052, disabling compactions & flushes 2024-11-26T20:14:22,325 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:22,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:22,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. after waiting 0 ms 2024-11-26T20:14:22,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:22,325 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:22,325 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 10a9d36c52dee7a92acb00483dfe6052: Waiting for close lock at 1732652062325Disabling compacts and flushes for region at 1732652062325Disabling writes for close at 1732652062325Writing region close event to WAL at 1732652062325Closed at 1732652062325 2024-11-26T20:14:22,326 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f9fa8bd8119f33b0d6fffe6254be9f4b, NAME => 'test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741860_1036 (size=45) 2024-11-26T20:14:22,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741860_1036 (size=45) 2024-11-26T20:14:22,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741860_1036 (size=45) 2024-11-26T20:14:22,369 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,369 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0880a54a28990b0c067cdb9447a1593b, disabling compactions & flushes 2024-11-26T20:14:22,369 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:22,370 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:22,370 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. after waiting 0 ms 2024-11-26T20:14:22,370 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:22,370 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:22,370 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0880a54a28990b0c067cdb9447a1593b: Waiting for close lock at 1732652062369Disabling compacts and flushes for region at 1732652062369Disabling writes for close at 1732652062370 (+1 ms)Writing region close event to WAL at 1732652062370Closed at 1732652062370 2024-11-26T20:14:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:22,371 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 6500def2f64bee0d698a3c7fb6413e30, NAME => 'test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741861_1037 (size=45) 2024-11-26T20:14:22,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741861_1037 (size=45) 2024-11-26T20:14:22,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741861_1037 (size=45) 2024-11-26T20:14:22,409 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1732652061178.0a569d698819974eb6636a41f610467c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,409 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0a569d698819974eb6636a41f610467c, disabling compactions & flushes 2024-11-26T20:14:22,409 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:22,409 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:22,409 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. after waiting 0 ms 2024-11-26T20:14:22,409 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:22,409 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:22,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0a569d698819974eb6636a41f610467c: Waiting for close lock at 1732652062409Disabling compacts and flushes for region at 1732652062409Disabling writes for close at 1732652062409Writing region close event to WAL at 1732652062409Closed at 1732652062409 2024-11-26T20:14:22,411 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 965d8bc3f7f9b8218376c29c89db6db3, NAME => 'test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d 2024-11-26T20:14:22,412 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,413 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f9fa8bd8119f33b0d6fffe6254be9f4b, disabling compactions & flushes 2024-11-26T20:14:22,413 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:22,413 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:22,413 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. after waiting 0 ms 2024-11-26T20:14:22,413 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:22,413 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:22,413 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f9fa8bd8119f33b0d6fffe6254be9f4b: Waiting for close lock at 1732652062413Disabling compacts and flushes for region at 1732652062413Disabling writes for close at 1732652062413Writing region close event to WAL at 1732652062413Closed at 1732652062413 2024-11-26T20:14:22,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741862_1038 (size=45) 2024-11-26T20:14:22,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741862_1038 (size=45) 2024-11-26T20:14:22,481 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,481 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 6500def2f64bee0d698a3c7fb6413e30, disabling compactions & flushes 2024-11-26T20:14:22,481 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:22,482 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:22,482 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. after waiting 0 ms 2024-11-26T20:14:22,482 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:22,482 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:22,482 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 6500def2f64bee0d698a3c7fb6413e30: Waiting for close lock at 1732652062481Disabling compacts and flushes for region at 1732652062481Disabling writes for close at 1732652062482 (+1 ms)Writing region close event to WAL at 1732652062482Closed at 1732652062482 2024-11-26T20:14:22,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741862_1038 (size=45) 2024-11-26T20:14:22,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741863_1039 (size=42) 2024-11-26T20:14:22,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741863_1039 (size=42) 2024-11-26T20:14:22,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741863_1039 (size=42) 2024-11-26T20:14:22,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:22,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 965d8bc3f7f9b8218376c29c89db6db3, disabling compactions & flushes 2024-11-26T20:14:22,515 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:22,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:22,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. after waiting 0 ms 2024-11-26T20:14:22,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:22,515 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:22,515 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 965d8bc3f7f9b8218376c29c89db6db3: Waiting for close lock at 1732652062515Disabling compacts and flushes for region at 1732652062515Disabling writes for close at 1732652062515Writing region close event to WAL at 1732652062515Closed at 1732652062515 2024-11-26T20:14:22,530 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T20:14:22,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,542 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,543 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,545 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732652061178.87979a5e316fc73bab2f01285393a426.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,546 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,546 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,546 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,546 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,547 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,547 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,547 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,547 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732652061178.c156031df6143a0a099724d68db39599.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,547 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,548 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,548 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,548 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,550 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732652061178.0a569d698819974eb6636a41f610467c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,553 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,553 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,554 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732652062530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732652062530"}]},"ts":"1732652062530"} 2024-11-26T20:14:22,666 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-26T20:14:22,673 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T20:14:22,677 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732652062673"}]},"ts":"1732652062673"} 2024-11-26T20:14:22,686 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-26T20:14:22,687 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {76919af20f29=0} racks are {/default-rack=0} 2024-11-26T20:14:22,764 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T20:14:22,764 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T20:14:22,764 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-26T20:14:22,764 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:14:22,764 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:14:22,764 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T20:14:22,764 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:14:22,765 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:14:22,765 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T20:14:22,765 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T20:14:22,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5070f0e2d0547381b8d60e94c76cdaff, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b8a0db8a2157574a65d6d70ba9f15e1, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e22e09e90e1a05e430419c43da9c80c, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e78554385cc62e1f4a220f96a9d9207d, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=34cb9834732a35daa158cb86b8b3b142, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dde94c133aedae28d634a201ddb741f7, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87979a5e316fc73bab2f01285393a426, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2ca44b38774b13852ff96928138ba075, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0a569d698819974eb6636a41f610467c, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=340a1869e06658f75d054f357d9a5c66, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=78cd4b9200f83099c4095aeeffca9b21, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=726bb7d945255f0a9e4ebbb78c65b7d5, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba6d284e3fe19dfa6e13c7648e255291, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=39562e7939506cf96cc70158f24d8ea2, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0880a54a28990b0c067cdb9447a1593b, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6500def2f64bee0d698a3c7fb6413e30, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=965d8bc3f7f9b8218376c29c89db6db3, ASSIGN}] 2024-11-26T20:14:22,801 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=965d8bc3f7f9b8218376c29c89db6db3, ASSIGN 2024-11-26T20:14:22,809 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=965d8bc3f7f9b8218376c29c89db6db3, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:22,818 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6500def2f64bee0d698a3c7fb6413e30, ASSIGN 2024-11-26T20:14:22,824 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6500def2f64bee0d698a3c7fb6413e30, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:22,830 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:22,837 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN; state=OFFLINE, location=76919af20f29,38853,1732652055556; forceNewPlan=false, retain=false 2024-11-26T20:14:22,846 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0880a54a28990b0c067cdb9447a1593b, ASSIGN 2024-11-26T20:14:22,849 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0880a54a28990b0c067cdb9447a1593b, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:22,856 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:22,861 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN; state=OFFLINE, location=76919af20f29,38853,1732652055556; forceNewPlan=false, retain=false 2024-11-26T20:14:22,864 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=39562e7939506cf96cc70158f24d8ea2, ASSIGN 2024-11-26T20:14:22,869 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=39562e7939506cf96cc70158f24d8ea2, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:22,872 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, ASSIGN 2024-11-26T20:14:22,875 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:22,877 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:22,882 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN; state=OFFLINE, location=76919af20f29,38853,1732652055556; forceNewPlan=false, retain=false 2024-11-26T20:14:22,885 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba6d284e3fe19dfa6e13c7648e255291, ASSIGN 2024-11-26T20:14:22,893 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ba6d284e3fe19dfa6e13c7648e255291, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:22,901 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=726bb7d945255f0a9e4ebbb78c65b7d5, ASSIGN 2024-11-26T20:14:22,905 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=726bb7d945255f0a9e4ebbb78c65b7d5, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:22,909 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:22,911 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN; state=OFFLINE, location=76919af20f29,38853,1732652055556; forceNewPlan=false, retain=false 2024-11-26T20:14:22,916 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=78cd4b9200f83099c4095aeeffca9b21, ASSIGN 2024-11-26T20:14:22,919 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=78cd4b9200f83099c4095aeeffca9b21, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:22,921 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=340a1869e06658f75d054f357d9a5c66, ASSIGN 2024-11-26T20:14:22,924 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=340a1869e06658f75d054f357d9a5c66, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:22,926 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0a569d698819974eb6636a41f610467c, ASSIGN 2024-11-26T20:14:22,927 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0a569d698819974eb6636a41f610467c, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:22,929 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:22,931 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN; state=OFFLINE, location=76919af20f29,38853,1732652055556; forceNewPlan=false, retain=false 2024-11-26T20:14:22,932 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:22,934 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN; state=OFFLINE, location=76919af20f29,38853,1732652055556; forceNewPlan=false, retain=false 2024-11-26T20:14:22,941 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2ca44b38774b13852ff96928138ba075, ASSIGN 2024-11-26T20:14:22,943 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2ca44b38774b13852ff96928138ba075, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:22,945 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87979a5e316fc73bab2f01285393a426, ASSIGN 2024-11-26T20:14:22,948 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=87979a5e316fc73bab2f01285393a426, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:22,950 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dde94c133aedae28d634a201ddb741f7, ASSIGN 2024-11-26T20:14:22,952 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dde94c133aedae28d634a201ddb741f7, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:22,957 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=34cb9834732a35daa158cb86b8b3b142, ASSIGN 2024-11-26T20:14:22,963 INFO [76919af20f29:34091 {}] balancer.BaseLoadBalancer(388): Reassigned 19 regions. 19 retained the pre-restart assignment. 2024-11-26T20:14:22,966 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=34cb9834732a35daa158cb86b8b3b142, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:23,009 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6500def2f64bee0d698a3c7fb6413e30, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=0880a54a28990b0c067cdb9447a1593b, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,015 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6500def2f64bee0d698a3c7fb6413e30, ASSIGN because future has completed 2024-11-26T20:14:23,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=340a1869e06658f75d054f357d9a5c66, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=87979a5e316fc73bab2f01285393a426, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0880a54a28990b0c067cdb9447a1593b, ASSIGN because future has completed 2024-11-26T20:14:23,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6500def2f64bee0d698a3c7fb6413e30, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:23,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, ASSIGN because future has completed 2024-11-26T20:14:23,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=726bb7d945255f0a9e4ebbb78c65b7d5, ASSIGN because future has completed 2024-11-26T20:14:23,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=340a1869e06658f75d054f357d9a5c66, ASSIGN because future has completed 2024-11-26T20:14:23,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 340a1869e06658f75d054f357d9a5c66, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:23,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=87979a5e316fc73bab2f01285393a426, ASSIGN because future has completed 2024-11-26T20:14:23,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87979a5e316fc73bab2f01285393a426, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:23,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 726bb7d945255f0a9e4ebbb78c65b7d5, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:23,117 INFO [76919af20f29:34091 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T20:14:23,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=34cb9834732a35daa158cb86b8b3b142, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0c6f3b0d7be2eee5a98bcf417b7ba9e, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:23,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0880a54a28990b0c067cdb9447a1593b, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:23,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=34cb9834732a35daa158cb86b8b3b142, ASSIGN because future has completed 2024-11-26T20:14:23,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34cb9834732a35daa158cb86b8b3b142, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:23,213 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T20:14:23,222 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:23,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43089, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T20:14:23,224 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=39562e7939506cf96cc70158f24d8ea2, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:23,226 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:23,228 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=78cd4b9200f83099c4095aeeffca9b21, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:23,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=0a569d698819974eb6636a41f610467c, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:23,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=2ca44b38774b13852ff96928138ba075, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:23,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=965d8bc3f7f9b8218376c29c89db6db3, ASSIGN because future has completed 2024-11-26T20:14:23,238 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=dde94c133aedae28d634a201ddb741f7, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:23,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,241 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,242 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c156031df6143a0a099724d68db39599, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,246 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:23,264 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:23,265 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 87979a5e316fc73bab2f01285393a426, NAME => 'test,iii,1732652061178.87979a5e316fc73bab2f01285393a426.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-26T20:14:23,266 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,266 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1732652061178.87979a5e316fc73bab2f01285393a426.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,266 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,266 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN; state=OFFLINE, location=76919af20f29,38853,1732652055556; forceNewPlan=false, retain=false 2024-11-26T20:14:23,266 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 965d8bc3f7f9b8218376c29c89db6db3, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:23,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=39562e7939506cf96cc70158f24d8ea2, ASSIGN because future has completed 2024-11-26T20:14:23,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ba6d284e3fe19dfa6e13c7648e255291, ASSIGN because future has completed 2024-11-26T20:14:23,280 INFO [StoreOpener-87979a5e316fc73bab2f01285393a426-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=78cd4b9200f83099c4095aeeffca9b21, ASSIGN because future has completed 2024-11-26T20:14:23,290 INFO [StoreOpener-87979a5e316fc73bab2f01285393a426-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87979a5e316fc73bab2f01285393a426 columnFamilyName family 2024-11-26T20:14:23,290 DEBUG [StoreOpener-87979a5e316fc73bab2f01285393a426-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0a569d698819974eb6636a41f610467c, ASSIGN because future has completed 2024-11-26T20:14:23,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2ca44b38774b13852ff96928138ba075, ASSIGN because future has completed 2024-11-26T20:14:23,299 INFO [StoreOpener-87979a5e316fc73bab2f01285393a426-1 {}] regionserver.HStore(327): Store=87979a5e316fc73bab2f01285393a426/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,299 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,305 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dde94c133aedae28d634a201ddb741f7, ASSIGN because future has completed 2024-11-26T20:14:23,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN because future has completed 2024-11-26T20:14:23,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde94c133aedae28d634a201ddb741f7, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:23,312 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,316 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,316 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN because future has completed 2024-11-26T20:14:23,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN because future has completed 2024-11-26T20:14:23,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN because future has completed 2024-11-26T20:14:23,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN because future has completed 2024-11-26T20:14:23,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:23,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN because future has completed 2024-11-26T20:14:23,339 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:23,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:23,369 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/87979a5e316fc73bab2f01285393a426/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,373 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 87979a5e316fc73bab2f01285393a426; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70745144, jitterRate=0.054184794425964355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,373 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:23,375 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 87979a5e316fc73bab2f01285393a426: Running coprocessor pre-open hook at 1732652063268Writing region info on filesystem at 1732652063268Initializing all the Stores at 1732652063276 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063276Cleaning up temporary data from old regions at 1732652063316 (+40 ms)Running coprocessor post-open hooks at 1732652063373 (+57 ms)Region opened successfully at 1732652063374 (+1 ms) 2024-11-26T20:14:23,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:23,377 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732652061178.87979a5e316fc73bab2f01285393a426., pid=33, masterSystemTime=1732652063212 2024-11-26T20:14:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:23,394 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:23,394 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:23,395 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:23,395 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 340a1869e06658f75d054f357d9a5c66, NAME => 'test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-26T20:14:23,395 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,396 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,396 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,396 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,407 INFO [StoreOpener-340a1869e06658f75d054f357d9a5c66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,411 INFO [StoreOpener-340a1869e06658f75d054f357d9a5c66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 340a1869e06658f75d054f357d9a5c66 columnFamilyName family 2024-11-26T20:14:23,411 DEBUG [StoreOpener-340a1869e06658f75d054f357d9a5c66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=87979a5e316fc73bab2f01285393a426, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,412 INFO [StoreOpener-340a1869e06658f75d054f357d9a5c66-1 {}] regionserver.HStore(327): Store=340a1869e06658f75d054f357d9a5c66/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,413 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,414 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,416 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,417 INFO [76919af20f29:34091 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T20:14:23,417 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,417 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87979a5e316fc73bab2f01285393a426, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:23,425 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=87979a5e316fc73bab2f01285393a426. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,429 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,444 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/340a1869e06658f75d054f357d9a5c66/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,447 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,449 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 340a1869e06658f75d054f357d9a5c66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66490409, jitterRate=-0.009215697646141052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,449 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:23,450 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 340a1869e06658f75d054f357d9a5c66: Running coprocessor pre-open hook at 1732652063396Writing region info on filesystem at 1732652063396Initializing all the Stores at 1732652063406 (+10 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063406Cleaning up temporary data from old regions at 1732652063417 (+11 ms)Running coprocessor post-open hooks at 1732652063449 (+32 ms)Region opened successfully at 1732652063449 2024-11-26T20:14:23,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN because future has completed 2024-11-26T20:14:23,454 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66., pid=32, masterSystemTime=1732652063212 2024-11-26T20:14:23,457 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T20:14:23,463 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:23,464 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:23,464 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:23,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38373, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T20:14:23,464 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => c0c6f3b0d7be2eee5a98bcf417b7ba9e, NAME => 'test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-26T20:14:23,465 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,465 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,465 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,465 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-26T20:14:23,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 87979a5e316fc73bab2f01285393a426, server=76919af20f29,44765,1732652055399 in 371 msec 2024-11-26T20:14:23,482 INFO [StoreOpener-c0c6f3b0d7be2eee5a98bcf417b7ba9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,482 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:23,483 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 965d8bc3f7f9b8218376c29c89db6db3, NAME => 'test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-26T20:14:23,483 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,483 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,483 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,483 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=87979a5e316fc73bab2f01285393a426, ASSIGN in 705 msec 2024-11-26T20:14:23,486 INFO [StoreOpener-c0c6f3b0d7be2eee5a98bcf417b7ba9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0c6f3b0d7be2eee5a98bcf417b7ba9e columnFamilyName family 2024-11-26T20:14:23,486 DEBUG [StoreOpener-c0c6f3b0d7be2eee5a98bcf417b7ba9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,487 INFO [StoreOpener-c0c6f3b0d7be2eee5a98bcf417b7ba9e-1 {}] regionserver.HStore(327): Store=c0c6f3b0d7be2eee5a98bcf417b7ba9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,488 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,490 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,491 INFO [StoreOpener-965d8bc3f7f9b8218376c29c89db6db3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,491 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,495 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,496 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,496 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 340a1869e06658f75d054f357d9a5c66, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:23,499 INFO [StoreOpener-965d8bc3f7f9b8218376c29c89db6db3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 965d8bc3f7f9b8218376c29c89db6db3 columnFamilyName family 2024-11-26T20:14:23,499 DEBUG [StoreOpener-965d8bc3f7f9b8218376c29c89db6db3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,504 INFO [StoreOpener-965d8bc3f7f9b8218376c29c89db6db3-1 {}] regionserver.HStore(327): Store=965d8bc3f7f9b8218376c29c89db6db3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,505 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:23,505 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => c156031df6143a0a099724d68db39599, NAME => 'test,sss,1732652061178.c156031df6143a0a099724d68db39599.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-26T20:14:23,506 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,506 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,506 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,sss,1732652061178.c156031df6143a0a099724d68db39599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,506 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,506 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,508 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,511 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,514 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,514 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c0c6f3b0d7be2eee5a98bcf417b7ba9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,515 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,515 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened c0c6f3b0d7be2eee5a98bcf417b7ba9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58881988, jitterRate=-0.12259000539779663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,515 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,515 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:23,515 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for c0c6f3b0d7be2eee5a98bcf417b7ba9e: Running coprocessor pre-open hook at 1732652063466Writing region info on filesystem at 1732652063466Initializing all the Stores at 1732652063474 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063474Cleaning up temporary data from old regions at 1732652063496 (+22 ms)Running coprocessor post-open hooks at 1732652063515 (+19 ms)Region opened successfully at 1732652063515 2024-11-26T20:14:23,518 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e., pid=35, masterSystemTime=1732652063212 2024-11-26T20:14:23,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=18 2024-11-26T20:14:23,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 340a1869e06658f75d054f357d9a5c66, server=76919af20f29,44765,1732652055399 in 442 msec 2024-11-26T20:14:23,523 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:23,523 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:23,524 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:23,524 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 0880a54a28990b0c067cdb9447a1593b, NAME => 'test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-26T20:14:23,525 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,525 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,525 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,525 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,526 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,534 INFO [StoreOpener-c156031df6143a0a099724d68db39599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,534 INFO [StoreOpener-0880a54a28990b0c067cdb9447a1593b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,539 INFO [StoreOpener-c156031df6143a0a099724d68db39599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c156031df6143a0a099724d68db39599 columnFamilyName family 2024-11-26T20:14:23,539 DEBUG [StoreOpener-c156031df6143a0a099724d68db39599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,540 INFO [StoreOpener-0880a54a28990b0c067cdb9447a1593b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0880a54a28990b0c067cdb9447a1593b columnFamilyName family 2024-11-26T20:14:23,541 DEBUG [StoreOpener-0880a54a28990b0c067cdb9447a1593b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,541 INFO [StoreOpener-c156031df6143a0a099724d68db39599-1 {}] regionserver.HStore(327): Store=c156031df6143a0a099724d68db39599/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,541 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,541 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/965d8bc3f7f9b8218376c29c89db6db3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=340a1869e06658f75d054f357d9a5c66, ASSIGN in 755 msec 2024-11-26T20:14:23,542 INFO [StoreOpener-0880a54a28990b0c067cdb9447a1593b-1 {}] regionserver.HStore(327): Store=0880a54a28990b0c067cdb9447a1593b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,542 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,542 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 965d8bc3f7f9b8218376c29c89db6db3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60324263, jitterRate=-0.10109843313694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,542 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:23,543 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 965d8bc3f7f9b8218376c29c89db6db3: Running coprocessor pre-open hook at 1732652063484Writing region info on filesystem at 1732652063484Initializing all the Stores at 1732652063490 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063490Cleaning up temporary data from old regions at 1732652063515 (+25 ms)Running coprocessor post-open hooks at 1732652063542 (+27 ms)Region opened successfully at 1732652063542 2024-11-26T20:14:23,545 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,545 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3., pid=38, masterSystemTime=1732652063456 2024-11-26T20:14:23,545 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,546 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,546 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,547 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,547 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,548 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,548 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,552 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,555 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,556 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,556 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:23,556 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:23,556 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:23,557 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => dde94c133aedae28d634a201ddb741f7, NAME => 'test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-26T20:14:23,557 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,558 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,558 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,558 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0c6f3b0d7be2eee5a98bcf417b7ba9e, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:23,569 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:23,572 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c156031df6143a0a099724d68db39599/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,573 INFO [StoreOpener-dde94c133aedae28d634a201ddb741f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,573 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/0880a54a28990b0c067cdb9447a1593b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,574 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened c156031df6143a0a099724d68db39599; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69066148, jitterRate=0.029165804386138916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,574 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 0880a54a28990b0c067cdb9447a1593b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64597613, jitterRate=-0.03742055594921112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,574 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c156031df6143a0a099724d68db39599 2024-11-26T20:14:23,575 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:23,575 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for c156031df6143a0a099724d68db39599: Running coprocessor pre-open hook at 1732652063506Writing region info on filesystem at 1732652063506Initializing all the Stores at 1732652063509 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063509Cleaning up temporary data from old regions at 1732652063547 (+38 ms)Running coprocessor post-open hooks at 1732652063574 (+27 ms)Region opened successfully at 1732652063575 (+1 ms) 2024-11-26T20:14:23,575 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 0880a54a28990b0c067cdb9447a1593b: Running coprocessor pre-open hook at 1732652063525Writing region info on filesystem at 1732652063525Initializing all the Stores at 1732652063527 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063528 (+1 ms)Cleaning up temporary data from old regions at 1732652063548 (+20 ms)Running coprocessor post-open hooks at 1732652063575 (+27 ms)Region opened successfully at 1732652063575 2024-11-26T20:14:23,577 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732652061178.c156031df6143a0a099724d68db39599., pid=43, masterSystemTime=1732652063491 2024-11-26T20:14:23,578 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b., pid=36, masterSystemTime=1732652063212 2024-11-26T20:14:23,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 965d8bc3f7f9b8218376c29c89db6db3, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:23,581 INFO [StoreOpener-dde94c133aedae28d634a201ddb741f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dde94c133aedae28d634a201ddb741f7 columnFamilyName family 2024-11-26T20:14:23,581 DEBUG [StoreOpener-dde94c133aedae28d634a201ddb741f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,591 INFO [StoreOpener-dde94c133aedae28d634a201ddb741f7-1 {}] regionserver.HStore(327): Store=dde94c133aedae28d634a201ddb741f7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,592 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,594 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:23,594 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:23,594 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:23,595 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => f5d1c8271bbf4e7a65c1cc98ef8dc026, NAME => 'test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-26T20:14:23,595 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,595 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,595 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,595 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,597 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,597 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,598 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,598 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,600 INFO [StoreOpener-f5d1c8271bbf4e7a65c1cc98ef8dc026-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,602 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:23,603 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:23,603 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:23,603 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 6500def2f64bee0d698a3c7fb6413e30, NAME => 'test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-26T20:14:23,604 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,604 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,604 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,604 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,604 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,608 INFO [StoreOpener-f5d1c8271bbf4e7a65c1cc98ef8dc026-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5d1c8271bbf4e7a65c1cc98ef8dc026 columnFamilyName family 2024-11-26T20:14:23,608 DEBUG [StoreOpener-f5d1c8271bbf4e7a65c1cc98ef8dc026-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,609 INFO [StoreOpener-6500def2f64bee0d698a3c7fb6413e30-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=24 2024-11-26T20:14:23,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure c0c6f3b0d7be2eee5a98bcf417b7ba9e, server=76919af20f29,44765,1732652055399 in 435 msec 2024-11-26T20:14:23,612 INFO [StoreOpener-f5d1c8271bbf4e7a65c1cc98ef8dc026-1 {}] regionserver.HStore(327): Store=f5d1c8271bbf4e7a65c1cc98ef8dc026/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,612 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,615 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,618 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,622 INFO [StoreOpener-6500def2f64bee0d698a3c7fb6413e30-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6500def2f64bee0d698a3c7fb6413e30 columnFamilyName family 2024-11-26T20:14:23,623 DEBUG [StoreOpener-6500def2f64bee0d698a3c7fb6413e30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,623 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,625 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,626 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,628 INFO [StoreOpener-6500def2f64bee0d698a3c7fb6413e30-1 {}] regionserver.HStore(327): Store=6500def2f64bee0d698a3c7fb6413e30/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,628 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,629 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/dde94c133aedae28d634a201ddb741f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, ASSIGN in 842 msec 2024-11-26T20:14:23,633 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened dde94c133aedae28d634a201ddb741f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64430230, jitterRate=-0.03991475701332092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,633 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:23,633 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for dde94c133aedae28d634a201ddb741f7: Running coprocessor pre-open hook at 1732652063558Writing region info on filesystem at 1732652063558Initializing all the Stores at 1732652063570 (+12 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063570Cleaning up temporary data from old regions at 1732652063598 (+28 ms)Running coprocessor post-open hooks at 1732652063633 (+35 ms)Region opened successfully at 1732652063633 2024-11-26T20:14:23,634 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,638 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,638 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7., pid=39, masterSystemTime=1732652063456 2024-11-26T20:14:23,640 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,642 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,642 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,645 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0880a54a28990b0c067cdb9447a1593b, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:23,651 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:23,651 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:23,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c156031df6143a0a099724d68db39599, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,660 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,660 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f5d1c8271bbf4e7a65c1cc98ef8dc026/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,665 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened f5d1c8271bbf4e7a65c1cc98ef8dc026; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70405887, jitterRate=0.04912947118282318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,665 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:23,666 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for f5d1c8271bbf4e7a65c1cc98ef8dc026: Running coprocessor pre-open hook at 1732652063596Writing region info on filesystem at 1732652063596Initializing all the Stores at 1732652063598 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063599 (+1 ms)Cleaning up temporary data from old regions at 1732652063626 (+27 ms)Running coprocessor post-open hooks at 1732652063665 (+39 ms)Region opened successfully at 1732652063665 2024-11-26T20:14:23,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,38853,1732652055556 because future has completed 2024-11-26T20:14:23,669 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026., pid=41, masterSystemTime=1732652063491 2024-11-26T20:14:23,676 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/6500def2f64bee0d698a3c7fb6413e30/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,676 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:23,676 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:23,677 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:23,677 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 6500def2f64bee0d698a3c7fb6413e30; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60279353, jitterRate=-0.1017676442861557}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,677 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:23,677 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => c5249ac65133ea81926c57db86f9b90e, NAME => 'test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-26T20:14:23,677 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 6500def2f64bee0d698a3c7fb6413e30: Running coprocessor pre-open hook at 1732652063604Writing region info on filesystem at 1732652063604Initializing all the Stores at 1732652063607 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063607Cleaning up temporary data from old regions at 1732652063643 (+36 ms)Running coprocessor post-open hooks at 1732652063677 (+34 ms)Region opened successfully at 1732652063677 2024-11-26T20:14:23,677 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,677 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,677 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,678 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,685 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30., pid=31, masterSystemTime=1732652063212 2024-11-26T20:14:23,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=27 2024-11-26T20:14:23,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 0880a54a28990b0c067cdb9447a1593b, server=76919af20f29,44765,1732652055399 in 516 msec 2024-11-26T20:14:23,692 INFO [StoreOpener-c5249ac65133ea81926c57db86f9b90e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,695 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:23,695 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:23,695 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:23,695 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 726bb7d945255f0a9e4ebbb78c65b7d5, NAME => 'test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-26T20:14:23,696 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,696 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,696 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,696 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,701 INFO [StoreOpener-c5249ac65133ea81926c57db86f9b90e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5249ac65133ea81926c57db86f9b90e columnFamilyName family 2024-11-26T20:14:23,702 DEBUG [StoreOpener-c5249ac65133ea81926c57db86f9b90e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,703 INFO [StoreOpener-c5249ac65133ea81926c57db86f9b90e-1 {}] regionserver.HStore(327): Store=c5249ac65133ea81926c57db86f9b90e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,703 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,704 INFO [StoreOpener-726bb7d945255f0a9e4ebbb78c65b7d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0880a54a28990b0c067cdb9447a1593b, ASSIGN in 924 msec 2024-11-26T20:14:23,705 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,709 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,711 INFO [StoreOpener-726bb7d945255f0a9e4ebbb78c65b7d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 726bb7d945255f0a9e4ebbb78c65b7d5 columnFamilyName family 2024-11-26T20:14:23,711 DEBUG [StoreOpener-726bb7d945255f0a9e4ebbb78c65b7d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,711 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,711 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,711 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,717 INFO [StoreOpener-726bb7d945255f0a9e4ebbb78c65b7d5-1 {}] regionserver.HStore(327): Store=726bb7d945255f0a9e4ebbb78c65b7d5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,718 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,719 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6500def2f64bee0d698a3c7fb6413e30, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:23,721 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,721 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,722 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,723 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,723 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,723 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,728 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,38853,1732652055556 because future has completed 2024-11-26T20:14:23,737 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,738 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-26T20:14:23,742 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c5249ac65133ea81926c57db86f9b90e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 6500def2f64bee0d698a3c7fb6413e30, server=76919af20f29,44765,1732652055399 in 702 msec 2024-11-26T20:14:23,743 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened c5249ac65133ea81926c57db86f9b90e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67577469, jitterRate=0.006982758641242981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,743 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:23,744 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for c5249ac65133ea81926c57db86f9b90e: Running coprocessor pre-open hook at 1732652063678Writing region info on filesystem at 1732652063678Initializing all the Stores at 1732652063689 (+11 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063689Cleaning up temporary data from old regions at 1732652063711 (+22 ms)Running coprocessor post-open hooks at 1732652063743 (+32 ms)Region opened successfully at 1732652063743 2024-11-26T20:14:23,750 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e., pid=40, masterSystemTime=1732652063491 2024-11-26T20:14:23,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6500def2f64bee0d698a3c7fb6413e30, ASSIGN in 974 msec 2024-11-26T20:14:23,757 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/726bb7d945255f0a9e4ebbb78c65b7d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,762 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 726bb7d945255f0a9e4ebbb78c65b7d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60940340, jitterRate=-0.09191817045211792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,762 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:23,762 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 726bb7d945255f0a9e4ebbb78c65b7d5: Running coprocessor pre-open hook at 1732652063696Writing region info on filesystem at 1732652063696Initializing all the Stores at 1732652063701 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063701Cleaning up temporary data from old regions at 1732652063723 (+22 ms)Running coprocessor post-open hooks at 1732652063762 (+39 ms)Region opened successfully at 1732652063762 2024-11-26T20:14:23,764 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5., pid=34, masterSystemTime=1732652063212 2024-11-26T20:14:23,767 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:23,767 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:23,767 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:23,768 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => d1c0c8c5fd0d7b81fc270155b15f8b86, NAME => 'test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-26T20:14:23,768 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:23,768 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:23,769 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:23,769 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 34cb9834732a35daa158cb86b8b3b142, NAME => 'test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-26T20:14:23,769 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,769 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,770 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,770 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,768 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,772 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:23,772 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,772 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=15 2024-11-26T20:14:23,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,38853,1732652055556 in 417 msec 2024-11-26T20:14:23,779 INFO [StoreOpener-34cb9834732a35daa158cb86b8b3b142-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN in 1.0090 sec 2024-11-26T20:14:23,789 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,790 INFO [StoreOpener-d1c0c8c5fd0d7b81fc270155b15f8b86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,799 INFO [StoreOpener-d1c0c8c5fd0d7b81fc270155b15f8b86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1c0c8c5fd0d7b81fc270155b15f8b86 columnFamilyName family 2024-11-26T20:14:23,799 DEBUG [StoreOpener-d1c0c8c5fd0d7b81fc270155b15f8b86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 726bb7d945255f0a9e4ebbb78c65b7d5, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:23,801 INFO [StoreOpener-34cb9834732a35daa158cb86b8b3b142-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34cb9834732a35daa158cb86b8b3b142 columnFamilyName family 2024-11-26T20:14:23,802 DEBUG [StoreOpener-34cb9834732a35daa158cb86b8b3b142-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:23,806 INFO [StoreOpener-d1c0c8c5fd0d7b81fc270155b15f8b86-1 {}] regionserver.HStore(327): Store=d1c0c8c5fd0d7b81fc270155b15f8b86/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,806 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,809 INFO [StoreOpener-34cb9834732a35daa158cb86b8b3b142-1 {}] regionserver.HStore(327): Store=34cb9834732a35daa158cb86b8b3b142/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:23,810 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,810 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,812 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,813 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,815 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,815 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,815 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,817 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,817 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,819 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,823 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=21 2024-11-26T20:14:23,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 726bb7d945255f0a9e4ebbb78c65b7d5, server=76919af20f29,44765,1732652055399 in 697 msec 2024-11-26T20:14:23,832 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/34cb9834732a35daa158cb86b8b3b142/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,832 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,834 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/d1c0c8c5fd0d7b81fc270155b15f8b86/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:23,834 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 34cb9834732a35daa158cb86b8b3b142; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62052526, jitterRate=-0.07534530758857727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,834 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:23,834 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 34cb9834732a35daa158cb86b8b3b142: Running coprocessor pre-open hook at 1732652063770Writing region info on filesystem at 1732652063770Initializing all the Stores at 1732652063777 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063777Cleaning up temporary data from old regions at 1732652063817 (+40 ms)Running coprocessor post-open hooks at 1732652063834 (+17 ms)Region opened successfully at 1732652063834 2024-11-26T20:14:23,837 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened d1c0c8c5fd0d7b81fc270155b15f8b86; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75484289, jitterRate=0.1248035579919815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:23,837 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:23,837 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for d1c0c8c5fd0d7b81fc270155b15f8b86: Running coprocessor pre-open hook at 1732652063772Writing region info on filesystem at 1732652063772Initializing all the Stores at 1732652063785 (+13 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652063785Cleaning up temporary data from old regions at 1732652063815 (+30 ms)Running coprocessor post-open hooks at 1732652063837 (+22 ms)Region opened successfully at 1732652063837 2024-11-26T20:14:23,839 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142., pid=37, masterSystemTime=1732652063212 2024-11-26T20:14:23,846 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86., pid=42, masterSystemTime=1732652063491 2024-11-26T20:14:23,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=726bb7d945255f0a9e4ebbb78c65b7d5, ASSIGN in 1.0640 sec 2024-11-26T20:14:23,851 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=c5249ac65133ea81926c57db86f9b90e, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,851 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c5249ac65133ea81926c57db86f9b90e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,851 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,853 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:23,853 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:23,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,38853,1732652055556 because future has completed 2024-11-26T20:14:23,858 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:23,858 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:23,859 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:23,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,38853,1732652055556 because future has completed 2024-11-26T20:14:23,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=16 2024-11-26T20:14:23,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,38853,1732652055556 in 536 msec 2024-11-26T20:14:23,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN in 1.1180 sec 2024-11-26T20:14:23,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=20 2024-11-26T20:14:23,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,38853,1732652055556 in 545 msec 2024-11-26T20:14:23,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN in 1.1470 sec 2024-11-26T20:14:23,937 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:23,960 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:23,961 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,041 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,064 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,067 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,144 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,168 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,171 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,248 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,273 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,278 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,351 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,379 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,383 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,454 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,457 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T20:14:24,458 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T20:14:24,465 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-26T20:14:24,465 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-26T20:14:24,467 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T20:14:24,468 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T20:14:24,468 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T20:14:24,468 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T20:14:24,485 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,487 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,557 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,588 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,590 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,660 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,692 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,693 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,762 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,794 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,809 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,865 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,911 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,915 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:24,968 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,016 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,017 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,059 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T20:14:25,061 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-26T20:14:25,070 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,119 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,124 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,173 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,226 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,226 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,275 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,329 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,330 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,378 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:25,432 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,432 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,481 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,535 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,536 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,584 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,638 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,639 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,686 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,741 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,742 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,789 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,793 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:14:25,844 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,845 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,891 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,948 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,950 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:25,994 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,051 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,053 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,097 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,154 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,157 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,199 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,259 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,262 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,302 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,362 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,364 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,404 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,465 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,468 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,507 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,570 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,571 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,609 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,672 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,673 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,712 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,832 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,834 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,835 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,935 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,938 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:26,939 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,039 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,041 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,041 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,142 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,143 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,143 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,245 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,245 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,246 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,347 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,348 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,348 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,450 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,450 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,450 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,552 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,553 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,553 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,655 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,656 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,656 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,757 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,760 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,761 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,862 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,864 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,865 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,965 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,972 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:27,972 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,073 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,074 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,075 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,176 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,176 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,177 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,278 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,279 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,279 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,381 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,381 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,381 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,483 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,483 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,484 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,587 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,587 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,588 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,690 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,690 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,690 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,792 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,792 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,793 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,894 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,895 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:28,895 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,000 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,001 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,001 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,103 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,103 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,103 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,205 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,205 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,206 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,307 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,307 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,308 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,409 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,409 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,410 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:29,511 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,511 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,512 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,613 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,613 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,614 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,715 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,716 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,716 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,817 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,817 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,818 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,919 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,919 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:29,920 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,021 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,022 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,022 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,123 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,124 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,124 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,225 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,226 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,227 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,328 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,329 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,329 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,431 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,432 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,432 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,534 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,534 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,535 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,636 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,637 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,637 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,738 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,739 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,739 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,841 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,841 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,841 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,943 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,943 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:30,945 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,045 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,045 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,048 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,148 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,149 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,150 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,250 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,251 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,251 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,352 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,353 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,354 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,454 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,456 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,457 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,558 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,559 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,560 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,662 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,663 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,663 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,765 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,766 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,767 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,868 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,869 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,870 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,973 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,974 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:31,975 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,078 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,079 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,079 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,182 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,183 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,184 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,286 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,288 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,288 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,391 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,392 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,393 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,495 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,496 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,497 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,598 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,600 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,601 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:32,704 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,007 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,108 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,109 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,112 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,212 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,213 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,214 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,315 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,316 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,317 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,419 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,419 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,420 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,522 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,523 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,523 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,625 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,626 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,627 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,729 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,730 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,731 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,831 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,832 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,832 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,935 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,936 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:33,937 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,038 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,038 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=c156031df6143a0a099724d68db39599. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,039 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,105 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-33219498_22 at /127.0.0.1:44978 [Receiving block BP-1003607189-172.17.0.3-1732652041092:blk_1073741836_1012] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 10145ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data2/, blockId=1073741836, seqno=38 2024-11-26T20:14:34,108 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-33219498_22 at /127.0.0.1:46356 [Receiving block BP-1003607189-172.17.0.3-1732652041092:blk_1073741836_1012] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 10150ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data6/, blockId=1073741836, seqno=38 2024-11-26T20:14:34,105 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-68783769_22 at /127.0.0.1:46292 [Receiving block BP-1003607189-172.17.0.3-1732652041092:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 10153ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data6/, blockId=1073741830, seqno=217 2024-11-26T20:14:34,107 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-33219498_22 at /127.0.0.1:48366 [Receiving block BP-1003607189-172.17.0.3-1732652041092:blk_1073741836_1012] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 10145ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data4/, blockId=1073741836, seqno=38 2024-11-26T20:14:34,105 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-68783769_22 at /127.0.0.1:44928 [Receiving block BP-1003607189-172.17.0.3-1732652041092:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 10153ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data2/, blockId=1073741830, seqno=217 2024-11-26T20:14:34,114 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-68783769_22 at /127.0.0.1:48316 [Receiving block BP-1003607189-172.17.0.3-1732652041092:blk_1073741830_1006] {}] datanode.BlockReceiver(767): Slow BlockReceiver write data to disk cost: 10164ms (threshold=300ms), volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data4/, blockId=1073741830, seqno=217 2024-11-26T20:14:34,115 INFO [AsyncFSWAL-0-hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData-prefix:76919af20f29,34091,1732652053862 {}] wal.AbstractFSWAL(1368): Slow sync cost: 10164 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK], DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK], DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK]] 2024-11-26T20:14:34,117 INFO [AsyncFSWAL-0-hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d-prefix:76919af20f29,38853,1732652055556.meta {}] wal.AbstractFSWAL(1368): Slow sync cost: 10159 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK], DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK], DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK]] 2024-11-26T20:14:34,118 WARN [AsyncFSWAL-0-hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d-prefix:76919af20f29,38853,1732652055556.meta {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=10159 ms, threshold=10000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK], DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK], DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK]] 2024-11-26T20:14:34,119 DEBUG [regionserver/76919af20f29:0.logRoller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 76919af20f29%2C38853%2C1732652055556.meta:.meta(num 1732652060143) roll requested 2024-11-26T20:14:34,120 WARN [AsyncFSWAL-0-hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData-prefix:76919af20f29,34091,1732652053862 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=10164 ms, threshold=10000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK], DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK], DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK]] 2024-11-26T20:14:34,123 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38853 {}] ipc.RpcServer(592): (responseTooSlow): {"call":"Mutate(org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$MutateRequest)","starttimems":"1732652063951","responsesize":"2","method":"Mutate","param":"region= hbase:meta,,1, row=test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142.","processingtimems":10171,"blockbytesscanned":"0","fsreadtime":"0","client":"172.17.0.3:40865","queuetimems":0,"class":"MiniHBaseClusterRegionServer"} 2024-11-26T20:14:34,124 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL AsyncFSWAL 76919af20f29%2C34091%2C1732652053862:(num 1732652057039) roll requested 2024-11-26T20:14:34,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34cb9834732a35daa158cb86b8b3b142, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:34,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=23 2024-11-26T20:14:34,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,38853,1732652055556 in 10.7610 sec 2024-11-26T20:14:34,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN in 11.3780 sec 2024-11-26T20:14:34,154 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=965d8bc3f7f9b8218376c29c89db6db3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,154 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=34cb9834732a35daa158cb86b8b3b142. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,160 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/WALs/76919af20f29,34091,1732652053862/76919af20f29%2C34091%2C1732652053862.1732652074124, exclude list is [], retry=0 2024-11-26T20:14:34,161 DEBUG [regionserver/76919af20f29:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, exclude list is [], retry=0 2024-11-26T20:14:34,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=11 2024-11-26T20:14:34,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 34cb9834732a35daa158cb86b8b3b142, server=76919af20f29,44765,1732652055399 in 10.9700 sec 2024-11-26T20:14:34,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK] 2024-11-26T20:14:34,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK] 2024-11-26T20:14:34,171 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK] 2024-11-26T20:14:34,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=34cb9834732a35daa158cb86b8b3b142, ASSIGN in 11.3990 sec 2024-11-26T20:14:34,178 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure dde94c133aedae28d634a201ddb741f7, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:34,190 INFO [regionserver/76919af20f29:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta with entries=39, filesize=21.92 KB; new WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta 2024-11-26T20:14:34,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=30 2024-11-26T20:14:34,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 965d8bc3f7f9b8218376c29c89db6db3, server=76919af20f29,43267,1732652055155 in 10.9080 sec 2024-11-26T20:14:34,208 DEBUG [regionserver/76919af20f29:0.logRoller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46547:46547),(127.0.0.1/127.0.0.1:37737:37737),(127.0.0.1/127.0.0.1:33291:33291)] 2024-11-26T20:14:34,208 DEBUG [regionserver/76919af20f29:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta is not closed yet, will try archiving it next time 2024-11-26T20:14:34,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK] 2024-11-26T20:14:34,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK] 2024-11-26T20:14:34,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK] 2024-11-26T20:14:34,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=965d8bc3f7f9b8218376c29c89db6db3, ASSIGN in 11.4380 sec 2024-11-26T20:14:34,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741836_1012 (size=22458) 2024-11-26T20:14:34,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741836_1012 (size=22458) 2024-11-26T20:14:34,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741836_1012 (size=22458) 2024-11-26T20:14:34,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=12 2024-11-26T20:14:34,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure dde94c133aedae28d634a201ddb741f7, server=76919af20f29,43267,1732652055155 in 10.9050 sec 2024-11-26T20:14:34,232 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/WALs/76919af20f29,34091,1732652053862/76919af20f29%2C34091%2C1732652053862.1732652057039 with entries=244, filesize=124.41 KB; new WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/WALs/76919af20f29,34091,1732652053862/76919af20f29%2C34091%2C1732652053862.1732652074124 2024-11-26T20:14:34,234 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46547:46547),(127.0.0.1/127.0.0.1:33291:33291),(127.0.0.1/127.0.0.1:37737:37737)] 2024-11-26T20:14:34,234 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/WALs/76919af20f29,34091,1732652053862/76919af20f29%2C34091%2C1732652053862.1732652057039 is not closed yet, will try archiving it next time 2024-11-26T20:14:34,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dde94c133aedae28d634a201ddb741f7, ASSIGN in 11.4580 sec 2024-11-26T20:14:34,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:34,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741830_1006 (size=127406) 2024-11-26T20:14:34,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741830_1006 (size=127406) 2024-11-26T20:14:34,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741830_1006 (size=127406) 2024-11-26T20:14:34,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:34,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9fa8bd8119f33b0d6fffe6254be9f4b, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:34,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ca44b38774b13852ff96928138ba075, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:34,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a569d698819974eb6636a41f610467c, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:34,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78cd4b9200f83099c4095aeeffca9b21, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:34,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba6d284e3fe19dfa6e13c7648e255291, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:34,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 39562e7939506cf96cc70158f24d8ea2, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:34,316 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e78554385cc62e1f4a220f96a9d9207d, ASSIGN 2024-11-26T20:14:34,317 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e78554385cc62e1f4a220f96a9d9207d, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:34,319 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1e22e09e90e1a05e430419c43da9c80c, ASSIGN 2024-11-26T20:14:34,322 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1e22e09e90e1a05e430419c43da9c80c, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:34,325 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:34,329 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN; state=OFFLINE, location=76919af20f29,38853,1732652055556; forceNewPlan=false, retain=false 2024-11-26T20:14:34,333 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4b8a0db8a2157574a65d6d70ba9f15e1, ASSIGN 2024-11-26T20:14:34,335 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4b8a0db8a2157574a65d6d70ba9f15e1, ASSIGN; state=OFFLINE, location=76919af20f29,43267,1732652055155; forceNewPlan=false, retain=false 2024-11-26T20:14:34,338 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5070f0e2d0547381b8d60e94c76cdaff, ASSIGN 2024-11-26T20:14:34,341 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5070f0e2d0547381b8d60e94c76cdaff, ASSIGN; state=OFFLINE, location=76919af20f29,44765,1732652055399; forceNewPlan=false, retain=false 2024-11-26T20:14:34,412 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:34,412 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 10a9d36c52dee7a92acb00483dfe6052, NAME => 'test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-26T20:14:34,413 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,413 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,413 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,413 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,416 INFO [StoreOpener-10a9d36c52dee7a92acb00483dfe6052-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,425 INFO [StoreOpener-10a9d36c52dee7a92acb00483dfe6052-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10a9d36c52dee7a92acb00483dfe6052 columnFamilyName family 2024-11-26T20:14:34,425 DEBUG [StoreOpener-10a9d36c52dee7a92acb00483dfe6052-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,426 INFO [StoreOpener-10a9d36c52dee7a92acb00483dfe6052-1 {}] regionserver.HStore(327): Store=10a9d36c52dee7a92acb00483dfe6052/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,426 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,428 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,428 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,429 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,429 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,432 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,434 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:34,435 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 78cd4b9200f83099c4095aeeffca9b21, NAME => 'test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-26T20:14:34,435 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,435 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,435 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,435 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,441 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/10a9d36c52dee7a92acb00483dfe6052/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,442 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 10a9d36c52dee7a92acb00483dfe6052; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69200327, jitterRate=0.0311652272939682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,442 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:34,443 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 10a9d36c52dee7a92acb00483dfe6052: Running coprocessor pre-open hook at 1732652074413Writing region info on filesystem at 1732652074413Initializing all the Stores at 1732652074415 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074415Cleaning up temporary data from old regions at 1732652074429 (+14 ms)Running coprocessor post-open hooks at 1732652074442 (+13 ms)Region opened successfully at 1732652074443 (+1 ms) 2024-11-26T20:14:34,444 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052., pid=45, masterSystemTime=1732652074396 2024-11-26T20:14:34,447 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:34,447 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:34,447 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:34,447 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => f9fa8bd8119f33b0d6fffe6254be9f4b, NAME => 'test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-26T20:14:34,448 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,448 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,448 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=10a9d36c52dee7a92acb00483dfe6052, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:34,448 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,448 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,38853,1732652055556 because future has completed 2024-11-26T20:14:34,453 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=10a9d36c52dee7a92acb00483dfe6052. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=26 2024-11-26T20:14:34,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,38853,1732652055556 in 206 msec 2024-11-26T20:14:34,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN in 11.6900 sec 2024-11-26T20:14:34,464 INFO [StoreOpener-78cd4b9200f83099c4095aeeffca9b21-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,467 INFO [StoreOpener-78cd4b9200f83099c4095aeeffca9b21-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78cd4b9200f83099c4095aeeffca9b21 columnFamilyName family 2024-11-26T20:14:34,467 DEBUG [StoreOpener-78cd4b9200f83099c4095aeeffca9b21-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,468 INFO [StoreOpener-78cd4b9200f83099c4095aeeffca9b21-1 {}] regionserver.HStore(327): Store=78cd4b9200f83099c4095aeeffca9b21/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,468 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,468 INFO [76919af20f29:34091 {}] balancer.BaseLoadBalancer(388): Reassigned 5 regions. 5 retained the pre-restart assignment. 2024-11-26T20:14:34,469 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1e22e09e90e1a05e430419c43da9c80c, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:34,469 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,470 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:34,470 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e78554385cc62e1f4a220f96a9d9207d, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,471 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,471 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a7108f1289f21074490c53bbef06d630, regionState=OPENING, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:34,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1e22e09e90e1a05e430419c43da9c80c, ASSIGN because future has completed 2024-11-26T20:14:34,474 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e22e09e90e1a05e430419c43da9c80c, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:34,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5070f0e2d0547381b8d60e94c76cdaff, ASSIGN because future has completed 2024-11-26T20:14:34,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e78554385cc62e1f4a220f96a9d9207d, ASSIGN because future has completed 2024-11-26T20:14:34,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4b8a0db8a2157574a65d6d70ba9f15e1, ASSIGN because future has completed 2024-11-26T20:14:34,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN because future has completed 2024-11-26T20:14:34,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,38853,1732652055556}] 2024-11-26T20:14:34,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b8a0db8a2157574a65d6d70ba9f15e1, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:34,492 INFO [StoreOpener-f9fa8bd8119f33b0d6fffe6254be9f4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e78554385cc62e1f4a220f96a9d9207d, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:34,495 INFO [StoreOpener-f9fa8bd8119f33b0d6fffe6254be9f4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9fa8bd8119f33b0d6fffe6254be9f4b columnFamilyName family 2024-11-26T20:14:34,495 DEBUG [StoreOpener-f9fa8bd8119f33b0d6fffe6254be9f4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,496 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/78cd4b9200f83099c4095aeeffca9b21/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,497 INFO [StoreOpener-f9fa8bd8119f33b0d6fffe6254be9f4b-1 {}] regionserver.HStore(327): Store=f9fa8bd8119f33b0d6fffe6254be9f4b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,498 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 78cd4b9200f83099c4095aeeffca9b21; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73225977, jitterRate=0.09115208685398102}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,498 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:34,498 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 78cd4b9200f83099c4095aeeffca9b21: Running coprocessor pre-open hook at 1732652074436Writing region info on filesystem at 1732652074436Initializing all the Stores at 1732652074439 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074439Cleaning up temporary data from old regions at 1732652074471 (+32 ms)Running coprocessor post-open hooks at 1732652074498 (+27 ms)Region opened successfully at 1732652074498 2024-11-26T20:14:34,499 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,501 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21., pid=49, masterSystemTime=1732652074420 2024-11-26T20:14:34,501 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,503 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,504 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,504 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5070f0e2d0547381b8d60e94c76cdaff, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:34,509 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:34,509 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:34,509 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:34,509 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,509 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => ba6d284e3fe19dfa6e13c7648e255291, NAME => 'test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-26T20:14:34,510 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,510 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,510 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,510 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,528 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,531 INFO [StoreOpener-ba6d284e3fe19dfa6e13c7648e255291-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78cd4b9200f83099c4095aeeffca9b21, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:34,536 INFO [StoreOpener-ba6d284e3fe19dfa6e13c7648e255291-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba6d284e3fe19dfa6e13c7648e255291 columnFamilyName family 2024-11-26T20:14:34,536 DEBUG [StoreOpener-ba6d284e3fe19dfa6e13c7648e255291-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,537 INFO [StoreOpener-ba6d284e3fe19dfa6e13c7648e255291-1 {}] regionserver.HStore(327): Store=ba6d284e3fe19dfa6e13c7648e255291/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,538 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,540 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,540 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f9fa8bd8119f33b0d6fffe6254be9f4b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=19 2024-11-26T20:14:34,541 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 78cd4b9200f83099c4095aeeffca9b21, server=76919af20f29,43267,1732652055155 in 251 msec 2024-11-26T20:14:34,542 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,542 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,545 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened f9fa8bd8119f33b0d6fffe6254be9f4b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72573114, jitterRate=0.08142367005348206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,545 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:34,545 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for f9fa8bd8119f33b0d6fffe6254be9f4b: Running coprocessor pre-open hook at 1732652074449Writing region info on filesystem at 1732652074449Initializing all the Stores at 1732652074451 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074451Cleaning up temporary data from old regions at 1732652074504 (+53 ms)Running coprocessor post-open hooks at 1732652074545 (+41 ms)Region opened successfully at 1732652074545 2024-11-26T20:14:34,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=78cd4b9200f83099c4095aeeffca9b21, ASSIGN in 11.7730 sec 2024-11-26T20:14:34,546 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b., pid=46, masterSystemTime=1732652074396 2024-11-26T20:14:34,546 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,550 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:34,550 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:34,550 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:34,550 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 5632c367c5d9113b557fd742bcb2b62f, NAME => 'test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-26T20:14:34,550 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:34,551 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,551 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,551 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,551 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,553 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/ba6d284e3fe19dfa6e13c7648e255291/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9fa8bd8119f33b0d6fffe6254be9f4b, server=76919af20f29,38853,1732652055556 because future has completed 2024-11-26T20:14:34,554 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened ba6d284e3fe19dfa6e13c7648e255291; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66663377, jitterRate=-0.006638273596763611}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,554 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:34,554 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for ba6d284e3fe19dfa6e13c7648e255291: Running coprocessor pre-open hook at 1732652074511Writing region info on filesystem at 1732652074511Initializing all the Stores at 1732652074513 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074513Cleaning up temporary data from old regions at 1732652074542 (+29 ms)Running coprocessor post-open hooks at 1732652074554 (+12 ms)Region opened successfully at 1732652074554 2024-11-26T20:14:34,555 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,38853,1732652055556, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,556 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291., pid=50, masterSystemTime=1732652074420 2024-11-26T20:14:34,559 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:34,559 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:34,560 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:34,560 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a569d698819974eb6636a41f610467c, NAME => 'test,mmm,1732652061178.0a569d698819974eb6636a41f610467c.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-26T20:14:34,560 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,560 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,mmm,1732652061178.0a569d698819974eb6636a41f610467c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=28 2024-11-26T20:14:34,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure f9fa8bd8119f33b0d6fffe6254be9f4b, server=76919af20f29,38853,1732652055556 in 300 msec 2024-11-26T20:14:34,560 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,560 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,561 INFO [StoreOpener-5632c367c5d9113b557fd742bcb2b62f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,562 INFO [StoreOpener-0a569d698819974eb6636a41f610467c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,565 INFO [StoreOpener-5632c367c5d9113b557fd742bcb2b62f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5632c367c5d9113b557fd742bcb2b62f columnFamilyName family 2024-11-26T20:14:34,565 INFO [StoreOpener-0a569d698819974eb6636a41f610467c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a569d698819974eb6636a41f610467c columnFamilyName family 2024-11-26T20:14:34,565 DEBUG [StoreOpener-5632c367c5d9113b557fd742bcb2b62f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,565 DEBUG [StoreOpener-0a569d698819974eb6636a41f610467c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN in 11.7930 sec 2024-11-26T20:14:34,566 INFO [StoreOpener-5632c367c5d9113b557fd742bcb2b62f-1 {}] regionserver.HStore(327): Store=5632c367c5d9113b557fd742bcb2b62f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,566 INFO [StoreOpener-0a569d698819974eb6636a41f610467c-1 {}] regionserver.HStore(327): Store=0a569d698819974eb6636a41f610467c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,566 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,566 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,568 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,568 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,568 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,568 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,568 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,569 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,569 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,569 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,569 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,573 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,573 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba6d284e3fe19dfa6e13c7648e255291, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:34,577 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/0a569d698819974eb6636a41f610467c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,577 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5632c367c5d9113b557fd742bcb2b62f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,578 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 5632c367c5d9113b557fd742bcb2b62f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68285930, jitterRate=0.017539650201797485}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,578 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 0a569d698819974eb6636a41f610467c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60998607, jitterRate=-0.091049924492836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,578 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:34,578 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:34,578 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 0a569d698819974eb6636a41f610467c: Running coprocessor pre-open hook at 1732652074561Writing region info on filesystem at 1732652074561Initializing all the Stores at 1732652074562 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074562Cleaning up temporary data from old regions at 1732652074569 (+7 ms)Running coprocessor post-open hooks at 1732652074578 (+9 ms)Region opened successfully at 1732652074578 2024-11-26T20:14:34,578 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 5632c367c5d9113b557fd742bcb2b62f: Running coprocessor pre-open hook at 1732652074551Writing region info on filesystem at 1732652074551Initializing all the Stores at 1732652074555 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074555Cleaning up temporary data from old regions at 1732652074569 (+14 ms)Running coprocessor post-open hooks at 1732652074578 (+9 ms)Region opened successfully at 1732652074578 2024-11-26T20:14:34,580 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732652061178.0a569d698819974eb6636a41f610467c., pid=48, masterSystemTime=1732652074420 2024-11-26T20:14:34,580 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f., pid=44, masterSystemTime=1732652074396 2024-11-26T20:14:34,583 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:34,583 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:34,583 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:34,583 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ca44b38774b13852ff96928138ba075, NAME => 'test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-26T20:14:34,584 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,584 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,584 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,584 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=22 2024-11-26T20:14:34,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure ba6d284e3fe19dfa6e13c7648e255291, server=76919af20f29,43267,1732652055155 in 289 msec 2024-11-26T20:14:34,585 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:34,585 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:34,587 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5632c367c5d9113b557fd742bcb2b62f, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:34,589 INFO [StoreOpener-2ca44b38774b13852ff96928138ba075-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,591 INFO [StoreOpener-2ca44b38774b13852ff96928138ba075-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ca44b38774b13852ff96928138ba075 columnFamilyName family 2024-11-26T20:14:34,591 DEBUG [StoreOpener-2ca44b38774b13852ff96928138ba075-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,38853,1732652055556 because future has completed 2024-11-26T20:14:34,592 INFO [StoreOpener-2ca44b38774b13852ff96928138ba075-1 {}] regionserver.HStore(327): Store=2ca44b38774b13852ff96928138ba075/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,592 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,594 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ba6d284e3fe19dfa6e13c7648e255291, ASSIGN in 11.8220 sec 2024-11-26T20:14:34,594 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,595 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,596 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,598 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=10 2024-11-26T20:14:34,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,38853,1732652055556 in 360 msec 2024-11-26T20:14:34,602 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/2ca44b38774b13852ff96928138ba075/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,603 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 2ca44b38774b13852ff96928138ba075; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70067843, jitterRate=0.044092223048210144}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,603 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:34,604 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 2ca44b38774b13852ff96928138ba075: Running coprocessor pre-open hook at 1732652074585Writing region info on filesystem at 1732652074585Initializing all the Stores at 1732652074586 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074586Cleaning up temporary data from old regions at 1732652074596 (+10 ms)Running coprocessor post-open hooks at 1732652074603 (+7 ms)Region opened successfully at 1732652074603 2024-11-26T20:14:34,605 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075., pid=47, masterSystemTime=1732652074420 2024-11-26T20:14:34,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN in 11.8340 sec 2024-11-26T20:14:34,608 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=0a569d698819974eb6636a41f610467c, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,608 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:34,609 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:34,609 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:34,609 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 39562e7939506cf96cc70158f24d8ea2, NAME => 'test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-26T20:14:34,610 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,610 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,610 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,610 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,612 INFO [StoreOpener-39562e7939506cf96cc70158f24d8ea2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a569d698819974eb6636a41f610467c, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:34,615 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=2ca44b38774b13852ff96928138ba075, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,619 INFO [StoreOpener-39562e7939506cf96cc70158f24d8ea2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39562e7939506cf96cc70158f24d8ea2 columnFamilyName family 2024-11-26T20:14:34,619 DEBUG [StoreOpener-39562e7939506cf96cc70158f24d8ea2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,620 INFO [StoreOpener-39562e7939506cf96cc70158f24d8ea2-1 {}] regionserver.HStore(327): Store=39562e7939506cf96cc70158f24d8ea2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ca44b38774b13852ff96928138ba075, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:34,621 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,622 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,623 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,624 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,624 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=17 2024-11-26T20:14:34,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a569d698819974eb6636a41f610467c, server=76919af20f29,43267,1732652055155 in 345 msec 2024-11-26T20:14:34,626 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0a569d698819974eb6636a41f610467c, ASSIGN in 11.8570 sec 2024-11-26T20:14:34,633 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/39562e7939506cf96cc70158f24d8ea2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,634 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 39562e7939506cf96cc70158f24d8ea2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68481744, jitterRate=0.02045750617980957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,634 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:34,635 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 39562e7939506cf96cc70158f24d8ea2: Running coprocessor pre-open hook at 1732652074610Writing region info on filesystem at 1732652074610Initializing all the Stores at 1732652074611 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074612 (+1 ms)Cleaning up temporary data from old regions at 1732652074624 (+12 ms)Running coprocessor post-open hooks at 1732652074635 (+11 ms)Region opened successfully at 1732652074635 2024-11-26T20:14:34,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=14 2024-11-26T20:14:34,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ca44b38774b13852ff96928138ba075, server=76919af20f29,43267,1732652055155 in 363 msec 2024-11-26T20:14:34,639 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:34,639 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e22e09e90e1a05e430419c43da9c80c, NAME => 'test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-26T20:14:34,640 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,640 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,640 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,640 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,644 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2., pid=51, masterSystemTime=1732652074420 2024-11-26T20:14:34,646 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:34,646 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => a7108f1289f21074490c53bbef06d630, NAME => 'test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-26T20:14:34,647 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,647 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,647 INFO [StoreOpener-1e22e09e90e1a05e430419c43da9c80c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,647 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,647 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2ca44b38774b13852ff96928138ba075, ASSIGN in 11.8700 sec 2024-11-26T20:14:34,651 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:34,651 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:34,653 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,654 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:34,654 INFO [StoreOpener-1e22e09e90e1a05e430419c43da9c80c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e22e09e90e1a05e430419c43da9c80c columnFamilyName family 2024-11-26T20:14:34,654 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => e78554385cc62e1f4a220f96a9d9207d, NAME => 'test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-26T20:14:34,654 DEBUG [StoreOpener-1e22e09e90e1a05e430419c43da9c80c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,655 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,655 INFO [StoreOpener-a7108f1289f21074490c53bbef06d630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,655 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,655 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,655 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,655 INFO [StoreOpener-1e22e09e90e1a05e430419c43da9c80c-1 {}] regionserver.HStore(327): Store=1e22e09e90e1a05e430419c43da9c80c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,656 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,658 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,659 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,660 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,660 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,663 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,670 INFO [StoreOpener-e78554385cc62e1f4a220f96a9d9207d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,670 INFO [StoreOpener-a7108f1289f21074490c53bbef06d630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7108f1289f21074490c53bbef06d630 columnFamilyName family 2024-11-26T20:14:34,671 DEBUG [StoreOpener-a7108f1289f21074490c53bbef06d630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,671 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=39562e7939506cf96cc70158f24d8ea2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,671 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/1e22e09e90e1a05e430419c43da9c80c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 39562e7939506cf96cc70158f24d8ea2, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:34,672 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 1e22e09e90e1a05e430419c43da9c80c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65987334, jitterRate=-0.016712099313735962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,673 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:34,673 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 1e22e09e90e1a05e430419c43da9c80c: Running coprocessor pre-open hook at 1732652074641Writing region info on filesystem at 1732652074641Initializing all the Stores at 1732652074647 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074647Cleaning up temporary data from old regions at 1732652074660 (+13 ms)Running coprocessor post-open hooks at 1732652074673 (+13 ms)Region opened successfully at 1732652074673 2024-11-26T20:14:34,673 INFO [StoreOpener-e78554385cc62e1f4a220f96a9d9207d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e78554385cc62e1f4a220f96a9d9207d columnFamilyName family 2024-11-26T20:14:34,673 DEBUG [StoreOpener-e78554385cc62e1f4a220f96a9d9207d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,674 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c., pid=52, masterSystemTime=1732652074630 2024-11-26T20:14:34,674 INFO [StoreOpener-e78554385cc62e1f4a220f96a9d9207d-1 {}] regionserver.HStore(327): Store=e78554385cc62e1f4a220f96a9d9207d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,674 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,675 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,676 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,676 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,676 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,676 INFO [StoreOpener-a7108f1289f21074490c53bbef06d630-1 {}] regionserver.HStore(327): Store=a7108f1289f21074490c53bbef06d630/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,677 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=25 2024-11-26T20:14:34,678 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 39562e7939506cf96cc70158f24d8ea2, server=76919af20f29,43267,1732652055155 in 366 msec 2024-11-26T20:14:34,678 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,679 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,679 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,682 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,690 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:34,690 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:34,690 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/a7108f1289f21074490c53bbef06d630/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,691 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:34,692 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 5070f0e2d0547381b8d60e94c76cdaff, NAME => 'test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-26T20:14:34,692 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened a7108f1289f21074490c53bbef06d630; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64206624, jitterRate=-0.04324674606323242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=39562e7939506cf96cc70158f24d8ea2, ASSIGN in 11.9100 sec 2024-11-26T20:14:34,692 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:34,692 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for a7108f1289f21074490c53bbef06d630: Running coprocessor pre-open hook at 1732652074648Writing region info on filesystem at 1732652074648Initializing all the Stores at 1732652074651 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074651Cleaning up temporary data from old regions at 1732652074679 (+28 ms)Running coprocessor post-open hooks at 1732652074692 (+13 ms)Region opened successfully at 1732652074692 2024-11-26T20:14:34,693 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,693 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,693 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,693 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,694 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:34,697 INFO [StoreOpener-5070f0e2d0547381b8d60e94c76cdaff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,697 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,698 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630., pid=53, masterSystemTime=1732652074635 2024-11-26T20:14:34,700 INFO [StoreOpener-5070f0e2d0547381b8d60e94c76cdaff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5070f0e2d0547381b8d60e94c76cdaff columnFamilyName family 2024-11-26T20:14:34,700 DEBUG [StoreOpener-5070f0e2d0547381b8d60e94c76cdaff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,701 INFO [StoreOpener-5070f0e2d0547381b8d60e94c76cdaff-1 {}] regionserver.HStore(327): Store=5070f0e2d0547381b8d60e94c76cdaff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,701 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,703 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,703 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,705 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,705 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e22e09e90e1a05e430419c43da9c80c, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:34,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a7108f1289f21074490c53bbef06d630, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,38853,1732652055556 2024-11-26T20:14:34,707 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/e78554385cc62e1f4a220f96a9d9207d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,704 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:34,709 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:34,710 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened e78554385cc62e1f4a220f96a9d9207d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71762320, jitterRate=0.06934189796447754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,710 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:34,710 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for e78554385cc62e1f4a220f96a9d9207d: Running coprocessor pre-open hook at 1732652074656Writing region info on filesystem at 1732652074656Initializing all the Stores at 1732652074658 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074658Cleaning up temporary data from old regions at 1732652074676 (+18 ms)Running coprocessor post-open hooks at 1732652074710 (+34 ms)Region opened successfully at 1732652074710 2024-11-26T20:14:34,710 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,711 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d., pid=55, masterSystemTime=1732652074640 2024-11-26T20:14:34,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,38853,1732652055556 because future has completed 2024-11-26T20:14:34,717 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:34,717 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:34,718 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:34,718 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b8a0db8a2157574a65d6d70ba9f15e1, NAME => 'test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-26T20:14:34,718 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,719 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:34,719 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,719 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=8 2024-11-26T20:14:34,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e22e09e90e1a05e430419c43da9c80c, server=76919af20f29,44765,1732652055399 in 240 msec 2024-11-26T20:14:34,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1e22e09e90e1a05e430419c43da9c80c, ASSIGN in 11.9580 sec 2024-11-26T20:14:34,731 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,733 INFO [StoreOpener-4b8a0db8a2157574a65d6d70ba9f15e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,734 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5070f0e2d0547381b8d60e94c76cdaff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,735 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 5070f0e2d0547381b8d60e94c76cdaff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72157700, jitterRate=0.07523351907730103}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,735 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:34,736 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 5070f0e2d0547381b8d60e94c76cdaff: Running coprocessor pre-open hook at 1732652074694Writing region info on filesystem at 1732652074694Initializing all the Stores at 1732652074696 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074696Cleaning up temporary data from old regions at 1732652074705 (+9 ms)Running coprocessor post-open hooks at 1732652074736 (+31 ms)Region opened successfully at 1732652074736 2024-11-26T20:14:34,738 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff., pid=56, masterSystemTime=1732652074630 2024-11-26T20:14:34,738 INFO [StoreOpener-4b8a0db8a2157574a65d6d70ba9f15e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b8a0db8a2157574a65d6d70ba9f15e1 columnFamilyName family 2024-11-26T20:14:34,738 DEBUG [StoreOpener-4b8a0db8a2157574a65d6d70ba9f15e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:34,739 INFO [StoreOpener-4b8a0db8a2157574a65d6d70ba9f15e1-1 {}] regionserver.HStore(327): Store=4b8a0db8a2157574a65d6d70ba9f15e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:34,740 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,742 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,743 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure e78554385cc62e1f4a220f96a9d9207d, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:34,744 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,744 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,745 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:34,745 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:34,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-11-26T20:14:34,748 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,38853,1732652055556 in 255 msec 2024-11-26T20:14:34,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN in 11.9800 sec 2024-11-26T20:14:34,753 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/4b8a0db8a2157574a65d6d70ba9f15e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T20:14:34,754 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 4b8a0db8a2157574a65d6d70ba9f15e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67719617, jitterRate=0.009100928902626038}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:34,754 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:34,755 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 4b8a0db8a2157574a65d6d70ba9f15e1: Running coprocessor pre-open hook at 1732652074719Writing region info on filesystem at 1732652074719Initializing all the Stores at 1732652074721 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652074721Cleaning up temporary data from old regions at 1732652074745 (+24 ms)Running coprocessor post-open hooks at 1732652074754 (+9 ms)Region opened successfully at 1732652074755 (+1 ms) 2024-11-26T20:14:34,756 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1., pid=54, masterSystemTime=1732652074640 2024-11-26T20:14:34,760 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:34,760 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:34,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-11-26T20:14:34,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure e78554385cc62e1f4a220f96a9d9207d, server=76919af20f29,43267,1732652055155 in 259 msec 2024-11-26T20:14:34,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e78554385cc62e1f4a220f96a9d9207d, ASSIGN in 11.9920 sec 2024-11-26T20:14:34,766 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:34,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b8a0db8a2157574a65d6d70ba9f15e1, server=76919af20f29,43267,1732652055155 because future has completed 2024-11-26T20:14:34,770 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, openSeqNum=2, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:34,774 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,43267,1732652055155, table=test, region=4b8a0db8a2157574a65d6d70ba9f15e1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5070f0e2d0547381b8d60e94c76cdaff, server=76919af20f29,44765,1732652055399 because future has completed 2024-11-26T20:14:34,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=6 2024-11-26T20:14:34,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b8a0db8a2157574a65d6d70ba9f15e1, server=76919af20f29,43267,1732652055155 in 289 msec 2024-11-26T20:14:34,780 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=76919af20f29,44765,1732652055399, table=test, region=5070f0e2d0547381b8d60e94c76cdaff. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-26T20:14:34,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4b8a0db8a2157574a65d6d70ba9f15e1, ASSIGN in 12.0120 sec 2024-11-26T20:14:34,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=5 2024-11-26T20:14:34,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5070f0e2d0547381b8d60e94c76cdaff, server=76919af20f29,44765,1732652055399 in 277 msec 2024-11-26T20:14:34,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T20:14:34,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5070f0e2d0547381b8d60e94c76cdaff, ASSIGN in 12.0240 sec 2024-11-26T20:14:34,801 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T20:14:34,802 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732652074802"}]},"ts":"1732652074802"} 2024-11-26T20:14:34,807 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-26T20:14:34,809 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T20:14:34,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 13.6190 sec 2024-11-26T20:14:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T20:14:39,518 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-26T20:14:39,518 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-26T20:14:39,519 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:14:39,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-26T20:14:39,537 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:14:39,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-26T20:14:39,538 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:14:39,543 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-26T20:14:39,563 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-26T20:14:39,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:14:39,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:14:39,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:14:39,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:14:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a7108f1289f21074490c53bbef06d630 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5632c367c5d9113b557fd742bcb2b62f changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f5d1c8271bbf4e7a65c1cc98ef8dc026 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c5249ac65133ea81926c57db86f9b90e changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d1c0c8c5fd0d7b81fc270155b15f8b86 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c156031df6143a0a099724d68db39599 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 10a9d36c52dee7a92acb00483dfe6052 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f9fa8bd8119f33b0d6fffe6254be9f4b changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4b8a0db8a2157574a65d6d70ba9f15e1 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e78554385cc62e1f4a220f96a9d9207d changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dde94c133aedae28d634a201ddb741f7 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2ca44b38774b13852ff96928138ba075 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0a569d698819974eb6636a41f610467c changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 78cd4b9200f83099c4095aeeffca9b21 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ba6d284e3fe19dfa6e13c7648e255291 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 39562e7939506cf96cc70158f24d8ea2 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 965d8bc3f7f9b8218376c29c89db6db3 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5070f0e2d0547381b8d60e94c76cdaff changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1e22e09e90e1a05e430419c43da9c80c changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 34cb9834732a35daa158cb86b8b3b142 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 87979a5e316fc73bab2f01285393a426 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 340a1869e06658f75d054f357d9a5c66 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 726bb7d945255f0a9e4ebbb78c65b7d5 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c0c6f3b0d7be2eee5a98bcf417b7ba9e changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0880a54a28990b0c067cdb9447a1593b changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6500def2f64bee0d698a3c7fb6413e30 changed from -1.0 to 0.0, refreshing cache 2024-11-26T20:14:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-26T20:14:39,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-26T20:14:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-26T20:14:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(204): Hosts are {76919af20f29=0} racks are {/default-rack=0} 2024-11-26T20:14:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-26T20:14:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-26T20:14:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-26T20:14:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:14:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:14:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-26T20:14:39,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:14:39,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:14:39,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-26T20:14:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-26T20:14:39,631 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-26T20:14:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-26T20:14:39,633 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:14:39,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-26T20:14:39,656 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=379, OpenFileDescriptor=617, MaxFileDescriptor=1048576, SystemLoadAverage=1352, ProcessCount=11, AvailableMemoryMB=294 2024-11-26T20:14:39,659 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-26T20:14:39,660 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-26T20:14:39,660 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 76919af20f29,38853,1732652055556 2024-11-26T20:14:39,660 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 76919af20f29,38853,1732652055556: Simulated kill ***** 2024-11-26T20:14:39,661 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-26T20:14:39,662 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-26T20:14:39,663 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-26T20:14:39,663 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-26T20:14:39,663 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-26T20:14:39,663 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 421040096 }, "NonHeapMemoryUsage": { "committed": 157614080, "init": 7667712, "max": -1, "used": 152768872 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-11-26T20:14:39,666 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34091 {}] master.MasterRpcServices(700): 76919af20f29,38853,1732652055556 reported a fatal error: ***** ABORTING region server 76919af20f29,38853,1732652055556: Simulated kill ***** 2024-11-26T20:14:39,667 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '76919af20f29,38853,1732652055556' ***** 2024-11-26T20:14:39,667 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-26T20:14:39,668 INFO [RS:2;76919af20f29:38853 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:14:39,668 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:14:39,668 INFO [RS:2;76919af20f29:38853 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-26T20:14:39,668 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:14:39,668 INFO [RS:2;76919af20f29:38853 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-26T20:14:39,668 INFO [RS:2;76919af20f29:38853 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:14:39,669 INFO [RS:2;76919af20f29:38853 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;76919af20f29:38853. 2024-11-26T20:14:39,669 DEBUG [RS:2;76919af20f29:38853 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:14:39,674 DEBUG [RS:2;76919af20f29:38853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:39,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741864_1040 (size=9749) 2024-11-26T20:14:39,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741864_1040 (size=9749) 2024-11-26T20:14:39,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741864_1040 (size=9749) 2024-11-26T20:14:39,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741833_1009 (size=2237) 2024-11-26T20:14:39,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741833_1009 (size=2237) 2024-11-26T20:14:39,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741833_1009 (size=2237) 2024-11-26T20:14:39,683 DEBUG [RS:2;76919af20f29:38853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:39,683 INFO [RS:2;76919af20f29:38853 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:14:39,683 INFO [RS:2;76919af20f29:38853 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38853 2024-11-26T20:14:39,742 INFO [regionserver/76919af20f29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:14:39,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/76919af20f29,38853,1732652055556 2024-11-26T20:14:39,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:14:39,915 INFO [RS:2;76919af20f29:38853 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:14:39,955 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T20:14:40,058 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [76919af20f29,38853,1732652055556] 2024-11-26T20:14:40,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:14:40,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38853-0x10178c82cdf0003, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:14:40,159 INFO [RS:2;76919af20f29:38853 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:14:40,159 INFO [RS:2;76919af20f29:38853 {}] regionserver.HRegionServer(1031): Exiting; stopping=76919af20f29,38853,1732652055556; zookeeper connection closed. 2024-11-26T20:14:40,159 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@45acd45 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@45acd45 2024-11-26T20:14:40,264 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/76919af20f29,38853,1732652055556 already deleted, retry=false 2024-11-26T20:14:40,264 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 76919af20f29,38853,1732652055556 on 76919af20f29,34091,1732652053862 2024-11-26T20:14:40,269 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:40,273 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 76919af20f29,38853,1732652055556 (carryingMeta=true) 76919af20f29,38853,1732652055556/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@22896d4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-26T20:14:40,274 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:40,275 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true, isMeta: true 2024-11-26T20:14:40,277 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting 2024-11-26T20:14:40,278 INFO [PEWorker-1 {}] master.SplitWALManager(105): 76919af20f29,38853,1732652055556 WAL count=2, meta=true 2024-11-26T20:14:40,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta}, {pid=59, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta}] 2024-11-26T20:14:40,285 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:40,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:40,292 INFO [regionserver/76919af20f29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T20:14:40,293 INFO [regionserver/76919af20f29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T20:14:40,296 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:40,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:40,370 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T20:14:40,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43267 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=60 2024-11-26T20:14:40,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44765 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=61 2024-11-26T20:14:40,470 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] wal.WALSplitter(299): Splitting hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, size=21.9 K (22458bytes) 2024-11-26T20:14:40,471 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-26T20:14:40,472 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-26T20:14:40,472 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta 2024-11-26T20:14:40,475 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] wal.WALSplitter(299): Splitting hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, size=9.5 K (9749bytes) 2024-11-26T20:14:40,475 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta 2024-11-26T20:14:40,477 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta after 3ms 2024-11-26T20:14:40,477 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta after 1ms 2024-11-26T20:14:40,486 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] wal.WALSplitter(310): Open hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta took 17ms 2024-11-26T20:14:40,486 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] wal.WALSplitter(310): Open hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta took 11ms 2024-11-26T20:14:40,498 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-26T20:14:40,498 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-26T20:14:40,504 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta so closing down 2024-11-26T20:14:40,504 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T20:14:40,505 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta so closing down 2024-11-26T20:14:40,505 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T20:14:40,506 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta.temp 2024-11-26T20:14:40,506 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000042-76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta.temp 2024-11-26T20:14:40,509 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta.temp 2024-11-26T20:14:40,511 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000042-76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta.temp 2024-11-26T20:14:40,511 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T20:14:40,512 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T20:14:40,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741867_1043 (size=22217) 2024-11-26T20:14:40,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741867_1043 (size=22217) 2024-11-26T20:14:40,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741867_1043 (size=22217) 2024-11-26T20:14:40,528 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta.temp (wrote 38 edits, skipped 0 edits in 1 ms) 2024-11-26T20:14:40,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741866_1042 (size=9749) 2024-11-26T20:14:40,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741866_1042 (size=9749) 2024-11-26T20:14:40,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741866_1042 (size=9749) 2024-11-26T20:14:40,531 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta.temp to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000041 2024-11-26T20:14:40,534 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] wal.WALSplitter(425): Processed 39 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, size=21.9 K, length=22458, corrupted=false, cancelled=false 2024-11-26T20:14:40,534 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] wal.WALSplitter(428): Completed split of hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, journal: Splitting hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, size=21.9 K (22458bytes) at 1732652080470Finishing writing output for hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta so closing down at 1732652080505 (+35 ms)Creating recovered edits writer path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta.temp at 1732652080509 (+4 ms)3 split writer threads finished at 1732652080512 (+3 ms)Closed recovered edits writer path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta.temp (wrote 38 edits, skipped 0 edits in 1 ms) at 1732652080529 (+17 ms)Rename recovered edits hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta.temp to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000041 at 1732652080531 (+2 ms)Processed 39 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, size=21.9 K, length=22458, corrupted=false, cancelled=false at 1732652080534 (+3 ms) 2024-11-26T20:14:40,535 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta 2024-11-26T20:14:40,536 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-26T20:14:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-26T20:14:40,546 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs 2024-11-26T20:14:40,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=58 2024-11-26T20:14:40,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 in 249 msec 2024-11-26T20:14:40,551 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:40,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 in 271 msec 2024-11-26T20:14:40,930 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000042-76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta.temp (wrote 19 edits, skipped 0 edits in 0 ms) 2024-11-26T20:14:40,933 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000042-76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta.temp to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T20:14:40,933 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] wal.WALSplitter(425): Processed 19 edits across 1 Regions in 442 ms; skipped=0; WAL=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, size=9.5 K, length=9749, corrupted=false, cancelled=false 2024-11-26T20:14:40,933 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] wal.WALSplitter(428): Completed split of hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, journal: Splitting hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, size=9.5 K (9749bytes) at 1732652080475Finishing writing output for hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta so closing down at 1732652080504 (+29 ms)Creating recovered edits writer path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000042-76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta.temp at 1732652080511 (+7 ms)3 split writer threads finished at 1732652080512 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000042-76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta.temp (wrote 19 edits, skipped 0 edits in 0 ms) at 1732652080930 (+418 ms)Rename recovered edits hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000042-76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta.temp to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732652080933 (+3 ms)Processed 19 edits across 1 Regions in 442 ms; skipped=0; WAL=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, size=9.5 K, length=9749, corrupted=false, cancelled=false at 1732652080933 2024-11-26T20:14:40,933 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta 2024-11-26T20:14:40,933 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-0 {event_type=RS_LOG_REPLAY, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-11-26T20:14:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.HMaster(4169): Remote procedure done, pid=60 2024-11-26T20:14:40,939 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs 2024-11-26T20:14:40,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-26T20:14:40,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 in 652 msec 2024-11-26T20:14:40,944 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:40,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-26T20:14:40,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 in 664 msec 2024-11-26T20:14:40,949 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting dir is empty, no logs to split. 2024-11-26T20:14:40,949 INFO [PEWorker-1 {}] master.SplitWALManager(105): 76919af20f29,38853,1732652055556 WAL count=0, meta=true 2024-11-26T20:14:40,950 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 76919af20f29,38853,1732652055556 WAL splitting is done? wals=0, meta=true 2024-11-26T20:14:40,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T20:14:40,953 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:40,954 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:41,110 DEBUG [76919af20f29:34091 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-26T20:14:41,110 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(204): Hosts are {76919af20f29=0} racks are {/default-rack=0} 2024-11-26T20:14:41,111 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T20:14:41,111 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T20:14:41,111 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:14:41,111 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:14:41,111 INFO [76919af20f29:34091 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:14:41,111 INFO [76919af20f29:34091 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:14:41,111 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T20:14:41,111 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:41,113 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPENING 2024-11-26T20:14:41,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:41,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:41,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:41,174 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=62, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:41,174 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:41,174 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:41,174 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:41,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:41,333 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T20:14:41,333 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-26T20:14:41,333 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-26T20:14:41,335 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=76919af20f29%2C43267%2C1732652055155.meta, suffix=.meta, logDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,43267,1732652055155, archiveDir=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs, maxLogs=32 2024-11-26T20:14:41,348 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,43267,1732652055155/76919af20f29%2C43267%2C1732652055155.meta.1732652081336.meta, exclude list is [], retry=0 2024-11-26T20:14:41,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34443,DS-a3350620-7d44-4208-a51a-24cd0df60136,DISK] 2024-11-26T20:14:41,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36117,DS-ebeab9c5-a6a6-4986-877c-eaff3583f349,DISK] 2024-11-26T20:14:41,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45553,DS-e1fe33c1-d0ee-431e-88af-ce0b59358dd9,DISK] 2024-11-26T20:14:41,354 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,43267,1732652055155/76919af20f29%2C43267%2C1732652055155.meta.1732652081336.meta 2024-11-26T20:14:41,354 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46547:46547),(127.0.0.1/127.0.0.1:33291:33291),(127.0.0.1/127.0.0.1:37737:37737)] 2024-11-26T20:14:41,354 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T20:14:41,354 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T20:14:41,354 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T20:14:41,355 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T20:14:41,355 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T20:14:41,355 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:41,355 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T20:14:41,355 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T20:14:41,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T20:14:41,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T20:14:41,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:41,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:41,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T20:14:41,360 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T20:14:41,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:41,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:41,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T20:14:41,362 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T20:14:41,362 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:41,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:41,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T20:14:41,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T20:14:41,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:41,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T20:14:41,365 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T20:14:41,366 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740 2024-11-26T20:14:41,369 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740 2024-11-26T20:14:41,370 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000041 2024-11-26T20:14:41,388 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(5793): Applied 183, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=41, path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000041 2024-11-26T20:14:41,390 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T20:14:41,397 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(5793): Applied 81, skipped 0, firstSequenceIdInLog=42, maxSequenceIdInLog=60, path=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T20:14:41,402 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-26T20:14:41,480 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/info/3008441cf11e4615b9b4b997d1cd0315 is 138, key is test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1./info:regioninfo/1732652074765/Put/seqid=0 2024-11-26T20:14:41,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741869_1045 (size=34924) 2024-11-26T20:14:41,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741869_1045 (size=34924) 2024-11-26T20:14:41,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741869_1045 (size=34924) 2024-11-26T20:14:41,894 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/info/3008441cf11e4615b9b4b997d1cd0315 2024-11-26T20:14:41,961 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/ns/1b3a22ebcad04492bf12f400153fbe82 is 43, key is default/ns:d/1732652060753/Put/seqid=0 2024-11-26T20:14:41,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741870_1046 (size=5153) 2024-11-26T20:14:41,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741870_1046 (size=5153) 2024-11-26T20:14:41,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741870_1046 (size=5153) 2024-11-26T20:14:41,969 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/ns/1b3a22ebcad04492bf12f400153fbe82 2024-11-26T20:14:41,994 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/table/d2e5a09180bc4cca8fd3967245463e35 is 40, key is test/table:state/1732652074802/Put/seqid=0 2024-11-26T20:14:42,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741871_1047 (size=5165) 2024-11-26T20:14:42,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741871_1047 (size=5165) 2024-11-26T20:14:42,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741871_1047 (size=5165) 2024-11-26T20:14:42,403 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/table/d2e5a09180bc4cca8fd3967245463e35 2024-11-26T20:14:42,414 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/info/3008441cf11e4615b9b4b997d1cd0315 as hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/info/3008441cf11e4615b9b4b997d1cd0315 2024-11-26T20:14:42,425 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/info/3008441cf11e4615b9b4b997d1cd0315, entries=260, sequenceid=60, filesize=34.1 K 2024-11-26T20:14:42,428 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/ns/1b3a22ebcad04492bf12f400153fbe82 as hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/ns/1b3a22ebcad04492bf12f400153fbe82 2024-11-26T20:14:42,437 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/ns/1b3a22ebcad04492bf12f400153fbe82, entries=2, sequenceid=60, filesize=5.0 K 2024-11-26T20:14:42,439 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/table/d2e5a09180bc4cca8fd3967245463e35 as hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/table/d2e5a09180bc4cca8fd3967245463e35 2024-11-26T20:14:42,449 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/table/d2e5a09180bc4cca8fd3967245463e35, entries=2, sequenceid=60, filesize=5.0 K 2024-11-26T20:14:42,455 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 1052ms, sequenceid=60, compaction requested=false; wal=null 2024-11-26T20:14:42,457 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000041 2024-11-26T20:14:42,458 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-26T20:14:42,460 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T20:14:42,460 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T20:14:42,461 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T20:14:42,462 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T20:14:42,465 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-26T20:14:42,467 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62312829, jitterRate=-0.07146649062633514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T20:14:42,467 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T20:14:42,467 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732652081355Writing region info on filesystem at 1732652081355Initializing all the Stores at 1732652081356 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652081356Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652081356Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652081356Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732652081356Obtaining lock to block concurrent updates at 1732652081402 (+46 ms)Preparing flush snapshotting stores in 1588230740 at 1732652081402Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732652081406 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732652081406Flushing 1588230740/info: creating writer at 1732652081408 (+2 ms)Flushing 1588230740/info: appending metadata at 1732652081470 (+62 ms)Flushing 1588230740/info: closing flushed file at 1732652081472 (+2 ms)Flushing 1588230740/ns: creating writer at 1732652081941 (+469 ms)Flushing 1588230740/ns: appending metadata at 1732652081960 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732652081960Flushing 1588230740/table: creating writer at 1732652081977 (+17 ms)Flushing 1588230740/table: appending metadata at 1732652081993 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732652081993Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66da7ce8: reopening flushed file at 1732652082413 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c2c8413: reopening flushed file at 1732652082426 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26080382: reopening flushed file at 1732652082437 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 1052ms, sequenceid=60, compaction requested=false; wal=null at 1732652082455 (+18 ms)Cleaning up temporary data from old regions at 1732652082460 (+5 ms)Running coprocessor post-open hooks at 1732652082467 (+7 ms)Region opened successfully at 1732652082467 2024-11-26T20:14:42,468 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=63, masterSystemTime=1732652081328 2024-11-26T20:14:42,471 DEBUG [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T20:14:42,471 INFO [RS_OPEN_META-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_META, pid=63}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T20:14:42,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:42,472 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:42,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:42,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:42,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:42,493 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=62, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:42,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:42,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:42,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:42,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-26T20:14:42,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 in 1.3190 sec 2024-11-26T20:14:42,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-26T20:14:42,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.5450 sec 2024-11-26T20:14:42,500 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 76919af20f29,38853,1732652055556 had 9 regions 2024-11-26T20:14:42,502 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true, isMeta: false 2024-11-26T20:14:42,504 INFO [PEWorker-1 {}] master.SplitWALManager(105): 76919af20f29,38853,1732652055556 WAL count=1, meta=false 2024-11-26T20:14:42,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440}] 2024-11-26T20:14:42,507 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:42,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:42,625 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T20:14:42,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44765 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=65 2024-11-26T20:14:42,683 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(299): Splitting hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440, size=2.2 K (2237bytes) 2024-11-26T20:14:42,683 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440 2024-11-26T20:14:42,684 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440 after 1ms 2024-11-26T20:14:42,687 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(310): Open hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440 took 4ms 2024-11-26T20:14:42,690 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(352): Last flushed sequenceid for c156031df6143a0a099724d68db39599: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T20:14:42,692 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(352): Last flushed sequenceid for f5d1c8271bbf4e7a65c1cc98ef8dc026: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T20:14:42,693 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(352): Last flushed sequenceid for c5249ac65133ea81926c57db86f9b90e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T20:14:42,695 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(352): Last flushed sequenceid for d1c0c8c5fd0d7b81fc270155b15f8b86: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T20:14:42,697 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(352): Last flushed sequenceid for 10a9d36c52dee7a92acb00483dfe6052: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T20:14:42,698 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(352): Last flushed sequenceid for f9fa8bd8119f33b0d6fffe6254be9f4b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T20:14:42,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(352): Last flushed sequenceid for 5632c367c5d9113b557fd742bcb2b62f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T20:14:42,701 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(352): Last flushed sequenceid for a7108f1289f21074490c53bbef06d630: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-26T20:14:42,701 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440 so closing down 2024-11-26T20:14:42,702 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-26T20:14:42,702 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.OutputSink(145): 3 split writer threads finished 2024-11-26T20:14:42,702 INFO [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 15 ms; skipped=8; WAL=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-26T20:14:42,702 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] wal.WALSplitter(428): Completed split of hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440, journal: Splitting hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440, size=2.2 K (2237bytes) at 1732652082683Finishing writing output for hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440 so closing down at 1732652082701 (+18 ms)3 split writer threads finished at 1732652082702 (+1 ms)Processed 8 edits across 0 Regions in 15 ms; skipped=8; WAL=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732652082702 2024-11-26T20:14:42,702 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440 2024-11-26T20:14:42,702 DEBUG [RS_LOG_REPLAY_OPS-regionserver/76919af20f29:0-1 {event_type=RS_LOG_REPLAY, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-26T20:14:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34091 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-26T20:14:42,707 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting/76919af20f29%2C38853%2C1732652055556.1732652059440 to hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs 2024-11-26T20:14:42,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-26T20:14:42,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 in 199 msec 2024-11-26T20:14:42,711 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:42,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=57 2024-11-26T20:14:42,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 in 207 msec 2024-11-26T20:14:42,716 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,38853,1732652055556-splitting dir is empty, no logs to split. 2024-11-26T20:14:42,716 INFO [PEWorker-1 {}] master.SplitWALManager(105): 76919af20f29,38853,1732652055556 WAL count=0, meta=false 2024-11-26T20:14:42,716 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 76919af20f29,38853,1732652055556 WAL splitting is done? wals=0, meta=false 2024-11-26T20:14:42,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN}, {pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN}] 2024-11-26T20:14:42,721 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:42,722 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:42,723 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:42,724 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:42,725 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:42,726 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:42,727 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:42,728 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:42,729 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:42,730 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:42,731 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:42,731 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:42,732 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:42,733 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:42,734 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:42,735 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-26T20:14:42,872 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(204): Hosts are {76919af20f29=0} racks are {/default-rack=0} 2024-11-26T20:14:42,873 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-26T20:14:42,873 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-26T20:14:42,873 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-26T20:14:42,873 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-26T20:14:42,873 INFO [76919af20f29:34091 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-26T20:14:42,873 INFO [76919af20f29:34091 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-26T20:14:42,873 DEBUG [76919af20f29:34091 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-26T20:14:42,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:42,875 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 76919af20f29:38853 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 76919af20f29/172.17.0.3:38853 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T20:14:42,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=76919af20f29,38853,1732652055556, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=76919af20f29,38853,1732652055556, seqNum=-1, error=java.net.ConnectException: Call to address=76919af20f29:38853 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 76919af20f29/172.17.0.3:38853 2024-11-26T20:14:42,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=76919af20f29,38853,1732652055556, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-26T20:14:42,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=76919af20f29,38853,1732652055556, seqNum=-1 from cache 2024-11-26T20:14:42,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 76919af20f29:38853 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 76919af20f29/172.17.0.3:38853 2024-11-26T20:14:42,881 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=c156031df6143a0a099724d68db39599, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:42,882 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T20:14:42,882 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=76919af20f29,43267,1732652055155, seqNum=-1] 2024-11-26T20:14:42,883 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T20:14:42,883 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:42,884 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, regionLocation=76919af20f29,44765,1732652055399 2024-11-26T20:14:42,884 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51585, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T20:14:42,886 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:42,887 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:42,888 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:42,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a7108f1289f21074490c53bbef06d630, regionState=OPENING, regionLocation=76919af20f29,43267,1732652055155 2024-11-26T20:14:42,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN because future has completed 2024-11-26T20:14:42,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:42,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN because future has completed 2024-11-26T20:14:42,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN because future has completed 2024-11-26T20:14:42,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN because future has completed 2024-11-26T20:14:42,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN because future has completed 2024-11-26T20:14:42,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:42,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN because future has completed 2024-11-26T20:14:42,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN because future has completed 2024-11-26T20:14:42,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:42,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:42,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155}] 2024-11-26T20:14:42,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:42,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:42,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN because future has completed 2024-11-26T20:14:42,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9fa8bd8119f33b0d6fffe6254be9f4b, server=76919af20f29,44765,1732652055399}] 2024-11-26T20:14:43,053 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:43,053 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => c156031df6143a0a099724d68db39599, NAME => 'test,sss,1732652061178.c156031df6143a0a099724d68db39599.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-26T20:14:43,053 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,053 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,sss,1732652061178.c156031df6143a0a099724d68db39599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:43,053 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,053 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,055 INFO [StoreOpener-c156031df6143a0a099724d68db39599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,056 INFO [StoreOpener-c156031df6143a0a099724d68db39599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c156031df6143a0a099724d68db39599 columnFamilyName family 2024-11-26T20:14:43,056 DEBUG [StoreOpener-c156031df6143a0a099724d68db39599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:43,057 INFO [StoreOpener-c156031df6143a0a099724d68db39599-1 {}] regionserver.HStore(327): Store=c156031df6143a0a099724d68db39599/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:43,057 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,058 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,060 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,060 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:43,060 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => d1c0c8c5fd0d7b81fc270155b15f8b86, NAME => 'test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-26T20:14:43,061 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,061 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,061 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,061 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:43,061 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,061 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,062 INFO [StoreOpener-d1c0c8c5fd0d7b81fc270155b15f8b86-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,063 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,064 INFO [StoreOpener-d1c0c8c5fd0d7b81fc270155b15f8b86-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1c0c8c5fd0d7b81fc270155b15f8b86 columnFamilyName family 2024-11-26T20:14:43,064 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened c156031df6143a0a099724d68db39599; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60112070, jitterRate=-0.10426035523414612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:43,064 DEBUG [StoreOpener-d1c0c8c5fd0d7b81fc270155b15f8b86-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:43,064 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c156031df6143a0a099724d68db39599 2024-11-26T20:14:43,064 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for c156031df6143a0a099724d68db39599: Running coprocessor pre-open hook at 1732652083054Writing region info on filesystem at 1732652083054Initializing all the Stores at 1732652083055 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652083055Cleaning up temporary data from old regions at 1732652083061 (+6 ms)Running coprocessor post-open hooks at 1732652083064 (+3 ms)Region opened successfully at 1732652083064 2024-11-26T20:14:43,065 INFO [StoreOpener-d1c0c8c5fd0d7b81fc270155b15f8b86-1 {}] regionserver.HStore(327): Store=d1c0c8c5fd0d7b81fc270155b15f8b86/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:43,065 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,065 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732652061178.c156031df6143a0a099724d68db39599., pid=74, masterSystemTime=1732652083049 2024-11-26T20:14:43,066 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,068 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:43,068 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:43,068 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:43,068 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 5632c367c5d9113b557fd742bcb2b62f, NAME => 'test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-26T20:14:43,069 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,069 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:43,069 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,069 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,070 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,070 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-26T20:14:43,070 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,071 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,071 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,071 INFO [StoreOpener-5632c367c5d9113b557fd742bcb2b62f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,071 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,073 INFO [StoreOpener-5632c367c5d9113b557fd742bcb2b62f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5632c367c5d9113b557fd742bcb2b62f columnFamilyName family 2024-11-26T20:14:43,073 DEBUG [StoreOpener-5632c367c5d9113b557fd742bcb2b62f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:43,073 INFO [StoreOpener-5632c367c5d9113b557fd742bcb2b62f-1 {}] regionserver.HStore(327): Store=5632c367c5d9113b557fd742bcb2b62f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:43,074 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,074 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,075 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,076 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened d1c0c8c5fd0d7b81fc270155b15f8b86; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62540124, jitterRate=-0.06807953119277954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:43,076 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:43,076 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for d1c0c8c5fd0d7b81fc270155b15f8b86: Running coprocessor pre-open hook at 1732652083061Writing region info on filesystem at 1732652083061Initializing all the Stores at 1732652083062 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652083062Cleaning up temporary data from old regions at 1732652083071 (+9 ms)Running coprocessor post-open hooks at 1732652083076 (+5 ms)Region opened successfully at 1732652083076 2024-11-26T20:14:43,077 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,077 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86., pid=75, masterSystemTime=1732652083056 2024-11-26T20:14:43,078 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,078 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,080 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:43,080 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:43,081 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:43,081 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => f5d1c8271bbf4e7a65c1cc98ef8dc026, NAME => 'test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-26T20:14:43,081 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,081 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,081 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:43,081 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,081 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,083 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 5632c367c5d9113b557fd742bcb2b62f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73600358, jitterRate=0.0967307984828949}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:43,083 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:43,083 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 5632c367c5d9113b557fd742bcb2b62f: Running coprocessor pre-open hook at 1732652083069Writing region info on filesystem at 1732652083069Initializing all the Stores at 1732652083071 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652083071Cleaning up temporary data from old regions at 1732652083078 (+7 ms)Running coprocessor post-open hooks at 1732652083083 (+5 ms)Region opened successfully at 1732652083083 2024-11-26T20:14:43,084 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f., pid=79, masterSystemTime=1732652083049 2024-11-26T20:14:43,084 INFO [StoreOpener-f5d1c8271bbf4e7a65c1cc98ef8dc026-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,086 INFO [StoreOpener-f5d1c8271bbf4e7a65c1cc98ef8dc026-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f5d1c8271bbf4e7a65c1cc98ef8dc026 columnFamilyName family 2024-11-26T20:14:43,087 DEBUG [StoreOpener-f5d1c8271bbf4e7a65c1cc98ef8dc026-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:43,084 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-26T20:14:43,088 INFO [StoreOpener-f5d1c8271bbf4e7a65c1cc98ef8dc026-1 {}] regionserver.HStore(327): Store=f5d1c8271bbf4e7a65c1cc98ef8dc026/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:43,088 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:43,088 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:43,088 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:43,088 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => c5249ac65133ea81926c57db86f9b90e, NAME => 'test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-26T20:14:43,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:43,089 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9fa8bd8119f33b0d6fffe6254be9f4b, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,089 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,089 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:43,089 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,089 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,090 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9fa8bd8119f33b0d6fffe6254be9f4b, server=76919af20f29,44765,1732652055399 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,091 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,092 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,092 INFO [StoreOpener-c5249ac65133ea81926c57db86f9b90e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,093 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,094 INFO [StoreOpener-c5249ac65133ea81926c57db86f9b90e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5249ac65133ea81926c57db86f9b90e columnFamilyName family 2024-11-26T20:14:43,094 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,094 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,094 DEBUG [StoreOpener-c5249ac65133ea81926c57db86f9b90e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:43,094 INFO [StoreOpener-c5249ac65133ea81926c57db86f9b90e-1 {}] regionserver.HStore(327): Store=c5249ac65133ea81926c57db86f9b90e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:43,094 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,095 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,096 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,096 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,097 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,097 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,097 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened f5d1c8271bbf4e7a65c1cc98ef8dc026; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70217139, jitterRate=0.046316906809806824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:43,097 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:43,097 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for f5d1c8271bbf4e7a65c1cc98ef8dc026: Running coprocessor pre-open hook at 1732652083081Writing region info on filesystem at 1732652083081Initializing all the Stores at 1732652083083 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652083083Cleaning up temporary data from old regions at 1732652083094 (+11 ms)Running coprocessor post-open hooks at 1732652083097 (+3 ms)Region opened successfully at 1732652083097 2024-11-26T20:14:43,098 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026., pid=77, masterSystemTime=1732652083056 2024-11-26T20:14:43,100 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,101 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:43,101 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:43,101 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:43,101 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 10a9d36c52dee7a92acb00483dfe6052, NAME => 'test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-26T20:14:43,101 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,101 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened c5249ac65133ea81926c57db86f9b90e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75438440, jitterRate=0.12412035465240479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:43,101 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:43,101 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:43,102 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,102 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for c5249ac65133ea81926c57db86f9b90e: Running coprocessor pre-open hook at 1732652083089Writing region info on filesystem at 1732652083089Initializing all the Stores at 1732652083091 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652083091Cleaning up temporary data from old regions at 1732652083097 (+6 ms)Running coprocessor post-open hooks at 1732652083101 (+4 ms)Region opened successfully at 1732652083101 2024-11-26T20:14:43,102 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,102 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e., pid=80, masterSystemTime=1732652083049 2024-11-26T20:14:43,104 INFO [StoreOpener-10a9d36c52dee7a92acb00483dfe6052-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,105 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:43,105 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:43,105 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:43,105 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => f9fa8bd8119f33b0d6fffe6254be9f4b, NAME => 'test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-26T20:14:43,105 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,105 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:43,105 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,106 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,106 INFO [StoreOpener-10a9d36c52dee7a92acb00483dfe6052-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10a9d36c52dee7a92acb00483dfe6052 columnFamilyName family 2024-11-26T20:14:43,106 DEBUG [StoreOpener-10a9d36c52dee7a92acb00483dfe6052-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:43,106 INFO [StoreOpener-10a9d36c52dee7a92acb00483dfe6052-1 {}] regionserver.HStore(327): Store=10a9d36c52dee7a92acb00483dfe6052/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:43,107 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,107 INFO [StoreOpener-f9fa8bd8119f33b0d6fffe6254be9f4b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,108 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,108 INFO [StoreOpener-f9fa8bd8119f33b0d6fffe6254be9f4b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9fa8bd8119f33b0d6fffe6254be9f4b columnFamilyName family 2024-11-26T20:14:43,108 DEBUG [StoreOpener-f9fa8bd8119f33b0d6fffe6254be9f4b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:43,109 INFO [StoreOpener-f9fa8bd8119f33b0d6fffe6254be9f4b-1 {}] regionserver.HStore(327): Store=f9fa8bd8119f33b0d6fffe6254be9f4b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:43,109 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,109 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,110 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,110 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,110 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,112 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,113 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,113 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 10a9d36c52dee7a92acb00483dfe6052; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62730947, jitterRate=-0.06523604691028595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:43,113 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:43,113 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 10a9d36c52dee7a92acb00483dfe6052: Running coprocessor pre-open hook at 1732652083102Writing region info on filesystem at 1732652083102Initializing all the Stores at 1732652083103 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652083103Cleaning up temporary data from old regions at 1732652083110 (+7 ms)Running coprocessor post-open hooks at 1732652083113 (+3 ms)Region opened successfully at 1732652083113 2024-11-26T20:14:43,114 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,115 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,115 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052., pid=78, masterSystemTime=1732652083056 2024-11-26T20:14:43,119 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:43,119 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:43,120 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:43,120 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => a7108f1289f21074490c53bbef06d630, NAME => 'test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-26T20:14:43,120 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,120 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T20:14:43,120 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,120 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,122 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,123 INFO [StoreOpener-a7108f1289f21074490c53bbef06d630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,124 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened f9fa8bd8119f33b0d6fffe6254be9f4b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61037639, jitterRate=-0.09046830236911774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:43,124 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:43,125 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for f9fa8bd8119f33b0d6fffe6254be9f4b: Running coprocessor pre-open hook at 1732652083106Writing region info on filesystem at 1732652083106Initializing all the Stores at 1732652083107 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652083107Cleaning up temporary data from old regions at 1732652083115 (+8 ms)Running coprocessor post-open hooks at 1732652083124 (+9 ms)Region opened successfully at 1732652083125 (+1 ms) 2024-11-26T20:14:43,125 INFO [StoreOpener-a7108f1289f21074490c53bbef06d630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a7108f1289f21074490c53bbef06d630 columnFamilyName family 2024-11-26T20:14:43,125 DEBUG [StoreOpener-a7108f1289f21074490c53bbef06d630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T20:14:43,126 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b., pid=81, masterSystemTime=1732652083049 2024-11-26T20:14:43,126 INFO [StoreOpener-a7108f1289f21074490c53bbef06d630-1 {}] regionserver.HStore(327): Store=a7108f1289f21074490c53bbef06d630/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T20:14:43,126 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,128 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,129 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:43,129 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:43,130 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,130 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,130 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,132 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,133 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened a7108f1289f21074490c53bbef06d630; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64308458, jitterRate=-0.04172930121421814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T20:14:43,133 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:43,133 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for a7108f1289f21074490c53bbef06d630: Running coprocessor pre-open hook at 1732652083121Writing region info on filesystem at 1732652083121Initializing all the Stores at 1732652083122 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732652083123 (+1 ms)Cleaning up temporary data from old regions at 1732652083130 (+7 ms)Running coprocessor post-open hooks at 1732652083133 (+3 ms)Region opened successfully at 1732652083133 2024-11-26T20:14:43,134 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630., pid=76, masterSystemTime=1732652083056 2024-11-26T20:14:43,136 DEBUG [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:43,136 INFO [RS_OPEN_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:43,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:43,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,175 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:43,175 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:43,176 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:43,176 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:43,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:43,176 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:43,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:43,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:43,186 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,187 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,188 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:43,190 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,190 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:43,192 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:43,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,198 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9fa8bd8119f33b0d6fffe6254be9f4b, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,199 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,199 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,200 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,200 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,201 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,201 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,201 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,201 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,201 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9fa8bd8119f33b0d6fffe6254be9f4b, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 24 msec 2024-11-26T20:14:43,205 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:43,206 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:43,206 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:43,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:43,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:43,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:43,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:43,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:43,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:43,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:43,283 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:43,284 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652113284 2024-11-26T20:14:43,284 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:43,284 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,286 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:43,287 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:43,287 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,287 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,44765,1732652055399 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,313 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,314 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,315 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,316 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,317 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,318 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:43,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652143318 2024-11-26T20:14:43,318 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652203318 2024-11-26T20:14:43,318 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-26T20:14:43,318 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:43,318 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,318 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:43,319 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:43,319 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:43,319 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:43,319 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:43,319 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:43,320 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:43,326 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:43,326 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,326 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,326 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,326 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,326 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,328 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,328 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,328 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:43,329 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,336 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,336 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,336 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,336 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,336 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,337 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,337 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5249ac65133ea81926c57db86f9b90e, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,338 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 17 msec 2024-11-26T20:14:43,339 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:43,339 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:43,339 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:43,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,360 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,361 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:43,361 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:43,361 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:43,361 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:43,361 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:43,361 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:43,361 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:43,361 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:43,364 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652113364 2024-11-26T20:14:43,364 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:43,364 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,366 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:43,368 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:43,368 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,369 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,377 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,379 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,380 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,382 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,385 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:43,385 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652143385 2024-11-26T20:14:43,386 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652203386 2024-11-26T20:14:43,386 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-11-26T20:14:43,386 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:43,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,386 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:43,386 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:43,386 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:43,387 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:43,387 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:43,387 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:43,388 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:43,392 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:43,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:43,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:43,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:43,399 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,400 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,401 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,401 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,402 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,402 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,402 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,402 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,402 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5632c367c5d9113b557fd742bcb2b62f, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-26T20:14:43,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:43,405 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:43,406 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:43,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,424 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:43,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:43,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:43,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:43,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:43,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:43,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:43,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:43,426 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652113426 2024-11-26T20:14:43,426 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:43,426 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,428 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:43,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,436 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:43,437 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,437 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,442 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:43,442 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652143442 2024-11-26T20:14:43,443 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652203443 2024-11-26T20:14:43,443 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-26T20:14:43,443 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:43,443 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,443 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:43,443 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:43,443 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:43,443 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:43,444 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:43,444 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:43,445 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T20:14:43,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:43,451 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,453 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,453 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:43,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,460 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a9d36c52dee7a92acb00483dfe6052, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,460 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-26T20:14:43,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:43,461 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:43,462 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:43,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:43,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:43,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:43,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:43,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:43,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:43,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:43,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:43,474 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652113474 2024-11-26T20:14:43,474 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:43,474 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:43,477 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:43,477 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,478 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,486 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:43,486 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652143486 2024-11-26T20:14:43,486 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652203486 2024-11-26T20:14:43,486 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-26T20:14:43,486 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:43,486 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,486 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:43,486 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:43,487 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:43,487 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:43,487 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:43,487 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:43,487 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,488 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,489 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T20:14:43,494 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:43,494 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:43,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:43,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:43,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,500 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,502 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,502 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,502 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure f5d1c8271bbf4e7a65c1cc98ef8dc026, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-26T20:14:43,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:43,505 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:43,505 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:43,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:43,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:43,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:43,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:43,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:43,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:43,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:43,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:43,520 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652113520 2024-11-26T20:14:43,520 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:43,520 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,521 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:43,523 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:43,523 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,523 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,528 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,529 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,530 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,531 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:43,531 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652143531 2024-11-26T20:14:43,531 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652203531 2024-11-26T20:14:43,531 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-26T20:14:43,531 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:43,531 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,531 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:43,532 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:43,532 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:43,532 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:43,532 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:43,532 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:43,533 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:43,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:43,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,538 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,539 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,540 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,540 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,540 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,540 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,540 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,540 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:43,541 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:43,543 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,544 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,545 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,545 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,546 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,546 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a7108f1289f21074490c53bbef06d630, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-26T20:14:43,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:43,548 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:43,548 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:43,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,560 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,560 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,562 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:43,562 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:43,562 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:43,562 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:43,563 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:43,563 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:43,563 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:43,563 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:43,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,564 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652113564 2024-11-26T20:14:43,564 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:43,564 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,566 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:43,573 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:43,573 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,573 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,577 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,577 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,581 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,582 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,583 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:43,583 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652143583 2024-11-26T20:14:43,583 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652203583 2024-11-26T20:14:43,583 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 18 msec 2024-11-26T20:14:43,583 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:43,583 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,583 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:43,583 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:43,584 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:43,584 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:43,584 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:43,584 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:43,591 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,592 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 7 msec 2024-11-26T20:14:43,595 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,597 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:43,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:43,598 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,601 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,602 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,603 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,603 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,604 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,604 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,604 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,604 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,604 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,604 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1c0c8c5fd0d7b81fc270155b15f8b86, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,605 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-26T20:14:43,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:43,613 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:43,613 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:43,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:43,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:43,625 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:43,625 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:43,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:43,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:43,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:43,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:43,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:43,626 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:43,656 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652113656 2024-11-26T20:14:43,656 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:43,656 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,658 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:43,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:43,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:43,670 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:43,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652143670 2024-11-26T20:14:43,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652203670 2024-11-26T20:14:43,670 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-26T20:14:43,677 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:43,677 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,677 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,920 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:43,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:43,921 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:43,921 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:43,921 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:43,921 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:43,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:43,922 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:43,924 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,925 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,928 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:43,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,929 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:43,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:43,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:43,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:43,933 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:43,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:43,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:43,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:43,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:43,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,946 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,947 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,948 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:43,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:43,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:43,948 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,948 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:43,949 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:43,949 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:43,949 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 24 msec 2024-11-26T20:14:43,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:43,953 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:43,953 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,011 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,011 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:44,011 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:44,011 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:44,011 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:44,011 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:44,012 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:44,012 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:44,025 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114025 2024-11-26T20:14:44,025 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,025 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,033 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,033 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:44,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,034 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,035 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,036 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,038 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,039 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,040 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,040 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144040 2024-11-26T20:14:44,040 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204040 2024-11-26T20:14:44,040 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-26T20:14:44,040 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,040 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,040 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,041 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,041 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,041 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,041 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,042 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,045 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,049 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,050 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,052 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,052 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,052 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,052 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:44,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,054 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,054 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,054 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,054 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,054 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,055 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,055 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,055 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:44,055 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,055 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:44,055 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,055 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN 2024-11-26T20:14:44,055 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-26T20:14:44,056 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,056 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,057 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,065 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,066 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:44,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:44,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:44,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:44,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:44,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:44,067 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5632c367c5d9113b557fd742bcb2b62f, ASSIGN to state=OFFLINE, location=null, table=test, region=5632c367c5d9113b557fd742bcb2b62f to restore RIT 2024-11-26T20:14:44,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,076 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114076 2024-11-26T20:14:44,076 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,076 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,077 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,081 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,082 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:44,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,082 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,090 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,090 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144090 2024-11-26T20:14:44,090 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204090 2024-11-26T20:14:44,090 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-26T20:14:44,090 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,091 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,091 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,091 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,092 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,092 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,092 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,092 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,097 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 4 msec 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,103 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,103 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:44,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN 2024-11-26T20:14:44,104 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-26T20:14:44,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,109 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,109 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:44,121 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:44,122 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:44,122 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:44,122 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:44,122 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c156031df6143a0a099724d68db39599, ASSIGN to state=OFFLINE, location=null, table=test, region=c156031df6143a0a099724d68db39599 to restore RIT 2024-11-26T20:14:44,125 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114125 2024-11-26T20:14:44,125 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,125 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,127 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,130 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,130 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:44,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,130 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,136 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,136 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144136 2024-11-26T20:14:44,136 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204136 2024-11-26T20:14:44,136 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-26T20:14:44,137 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,137 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,137 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,137 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,137 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,138 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,139 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T20:14:44,140 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,141 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,142 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,144 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,154 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,155 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,155 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,155 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,155 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,155 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,156 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,156 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,156 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,156 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN 2024-11-26T20:14:44,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-26T20:14:44,161 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,161 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,161 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,180 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,180 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,180 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:44,180 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:44,181 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:44,181 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:44,181 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b, ASSIGN to state=OFFLINE, location=null, table=test, region=f9fa8bd8119f33b0d6fffe6254be9f4b to restore RIT 2024-11-26T20:14:44,188 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114188 2024-11-26T20:14:44,188 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,190 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,192 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,193 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,193 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,194 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,197 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,199 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,199 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,199 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144199 2024-11-26T20:14:44,199 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204199 2024-11-26T20:14:44,199 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-26T20:14:44,199 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,199 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,199 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,200 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,200 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,200 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,200 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,200 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,201 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:44,203 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,204 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,205 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,206 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,207 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,207 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,208 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN 2024-11-26T20:14:44,208 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T20:14:44,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,209 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,210 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,223 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:44,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:44,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:44,224 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=a7108f1289f21074490c53bbef06d630, ASSIGN to state=OFFLINE, location=null, table=test, region=a7108f1289f21074490c53bbef06d630 to restore RIT 2024-11-26T20:14:44,232 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114232 2024-11-26T20:14:44,232 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,232 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,234 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,238 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,238 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,239 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,244 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,249 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,249 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144249 2024-11-26T20:14:44,249 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204249 2024-11-26T20:14:44,249 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-26T20:14:44,249 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,249 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,249 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,249 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,252 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,253 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,253 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,253 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,254 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,262 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,264 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,265 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,265 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN 2024-11-26T20:14:44,265 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-26T20:14:44,269 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,269 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,269 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,290 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,290 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:44,290 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:44,290 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=10a9d36c52dee7a92acb00483dfe6052, ASSIGN to state=OFFLINE, location=null, table=test, region=10a9d36c52dee7a92acb00483dfe6052 to restore RIT 2024-11-26T20:14:44,300 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114300 2024-11-26T20:14:44,300 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,300 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,302 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,309 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,310 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,312 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,312 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,312 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,312 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,313 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144313 2024-11-26T20:14:44,313 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204313 2024-11-26T20:14:44,313 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-26T20:14:44,313 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,313 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,313 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,313 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,314 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,314 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,314 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,314 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 9 msec 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,337 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,337 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,338 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,339 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,339 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,339 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,339 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,339 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,339 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,339 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,339 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN 2024-11-26T20:14:44,339 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-26T20:14:44,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,345 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,345 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,348 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,349 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,364 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,365 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,365 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:44,365 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026, ASSIGN to state=OFFLINE, location=null, table=test, region=f5d1c8271bbf4e7a65c1cc98ef8dc026 to restore RIT 2024-11-26T20:14:44,366 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,380 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114380 2024-11-26T20:14:44,380 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,382 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,384 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,391 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,392 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,392 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,392 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,392 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,393 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,394 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,394 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144394 2024-11-26T20:14:44,394 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204394 2024-11-26T20:14:44,396 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-26T20:14:44,396 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,396 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,396 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,396 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,397 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,397 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,397 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,397 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,398 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,401 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,402 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,402 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,402 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,403 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,405 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,406 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,406 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,406 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN 2024-11-26T20:14:44,406 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-26T20:14:44,425 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,425 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,425 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,467 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,469 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,502 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,502 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c5249ac65133ea81926c57db86f9b90e, ASSIGN to state=OFFLINE, location=null, table=test, region=c5249ac65133ea81926c57db86f9b90e to restore RIT 2024-11-26T20:14:44,503 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114503 2024-11-26T20:14:44,503 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,505 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,510 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,513 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,514 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,514 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144514 2024-11-26T20:14:44,514 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204514 2024-11-26T20:14:44,514 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-26T20:14:44,514 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,514 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,514 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,764 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,764 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,765 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,765 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,765 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,765 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,766 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,766 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T20:14:44,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN 2024-11-26T20:14:44,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-26T20:14:44,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,776 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,788 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,788 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86, ASSIGN to state=OFFLINE, location=null, table=test, region=d1c0c8c5fd0d7b81fc270155b15f8b86 to restore RIT 2024-11-26T20:14:44,789 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114789 2024-11-26T20:14:44,789 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,789 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,790 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,791 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,791 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,791 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,794 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,796 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,796 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144796 2024-11-26T20:14:44,796 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204796 2024-11-26T20:14:44,796 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T20:14:44,796 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,796 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,796 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,796 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,797 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,797 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,797 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,797 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=64, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,800 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,802 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,803 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=64, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,803 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T20:14:44,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,804 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,804 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,805 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,807 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,813 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,813 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,813 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,813 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114813 2024-11-26T20:14:44,814 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,814 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,815 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,817 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,817 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,817 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,819 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,820 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,821 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,821 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144821 2024-11-26T20:14:44,821 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204821 2024-11-26T20:14:44,821 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T20:14:44,821 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,821 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,821 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,821 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,822 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,822 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,822 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,823 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,824 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,827 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.1732652059440, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,827 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T20:14:44,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,828 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,828 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,840 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,843 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114843 2024-11-26T20:14:44,843 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,843 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,844 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,845 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,845 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,846 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,850 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,850 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144850 2024-11-26T20:14:44,850 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204850 2024-11-26T20:14:44,850 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T20:14:44,850 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,850 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,850 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,850 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,851 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,851 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,851 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,851 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,852 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:44,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,853 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-26T20:14:44,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,854 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,855 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,855 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-26T20:14:44,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,855 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,855 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,886 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,886 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,886 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,887 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114887 2024-11-26T20:14:44,887 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,887 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,888 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,889 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,889 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,889 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,892 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,894 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,895 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,895 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,895 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144895 2024-11-26T20:14:44,895 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204895 2024-11-26T20:14:44,895 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T20:14:44,895 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,895 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,896 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,896 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,896 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,896 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,896 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,896 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,897 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:44,899 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,899 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,899 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,899 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,899 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,899 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-26T20:14:44,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,900 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,901 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T20:14:44,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,902 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,903 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,914 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,915 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34091 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,918 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,918 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,918 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,919 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114919 2024-11-26T20:14:44,920 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,920 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,921 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,925 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,925 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,925 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,925 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,928 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,928 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144928 2024-11-26T20:14:44,928 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204928 2024-11-26T20:14:44,929 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T20:14:44,929 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,929 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,929 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,929 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,929 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,929 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,929 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,930 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,930 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:44,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,932 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,933 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,933 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,933 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,934 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,934 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,935 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,935 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T20:14:44,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,936 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,936 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:44,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:44,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:44,947 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652114947 2024-11-26T20:14:44,947 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:44,947 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,948 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:44,952 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:44,952 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,952 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:44,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:44,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:44,955 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:44,955 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652144955 2024-11-26T20:14:44,955 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652204955 2024-11-26T20:14:44,955 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T20:14:44,955 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:44,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:44,956 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:44,956 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:44,956 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:44,956 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:44,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:44,956 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:44,957 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:44,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:44,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:44,958 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=59, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-26T20:14:44,958 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,959 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=59, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:44,960 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-26T20:14:44,960 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:44,961 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:44,961 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:45,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,013 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,013 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,013 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,013 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652115013 2024-11-26T20:14:45,014 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:45,014 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:45,015 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:45,016 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:45,016 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:45,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:45,017 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:45,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,018 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,020 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,021 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,021 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,021 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,021 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:45,021 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652145021 2024-11-26T20:14:45,021 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652205021 2024-11-26T20:14:45,021 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-26T20:14:45,021 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:45,021 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:45,021 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:45,021 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:45,021 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:45,021 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:45,022 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:45,022 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:45,023 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:45,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:45,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:45,024 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:45,024 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:45,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:45,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:45,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:45,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 2024-11-26T20:14:45,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:45,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:45,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652074120.meta, worker=76919af20f29,43267,1732652055155 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:45,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-26T20:14:45,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:45,026 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:45,026 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:45,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652115034 2024-11-26T20:14:45,035 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:45,035 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:45,036 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:45,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,040 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,041 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:45,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,041 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:45,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:45,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,041 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:45,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,043 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:45,043 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652145043 2024-11-26T20:14:45,043 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652205043 2024-11-26T20:14:45,043 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T20:14:45,043 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:45,043 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:45,044 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:45,044 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:45,044 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:45,044 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:45,044 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:45,044 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:45,045 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T20:14:45,047 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:45,047 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-26T20:14:45,047 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:45,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:45,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:45,048 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 2024-11-26T20:14:45,048 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:45,048 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 76919af20f29%2C38853%2C1732652055556.meta.1732652060143.meta, worker=76919af20f29,44765,1732652055399 didn't hold the lock before restarting, skip acquiring lock. 2024-11-26T20:14:45,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-26T20:14:45,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:45,049 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:45,050 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:45,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,062 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652115062 2024-11-26T20:14:45,062 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:45,062 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:45,063 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:45,065 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-26T20:14:45,065 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:45,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:45,065 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,068 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:45,068 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652145068 2024-11-26T20:14:45,068 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652205068 2024-11-26T20:14:45,068 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-11-26T20:14:45,068 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-26T20:14:45,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:45,068 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:45,068 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:45,069 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-26T20:14:45,069 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-26T20:14:45,069 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-26T20:14:45,069 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T20:14:45,070 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T20:14:45,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-26T20:14:45,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T20:14:45,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-26T20:14:45,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true 2024-11-26T20:14:45,072 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-26T20:14:45,072 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-26T20:14:45,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=61 2024-11-26T20:14:45,073 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=76919af20f29,43267,1732652055155, table=hbase:meta, region=1588230740 2024-11-26T20:14:45,073 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 76919af20f29,43267,1732652055155, state=OPEN 2024-11-26T20:14:45,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T20:14:45,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,086 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T20:14:45,088 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732652115088 2024-11-26T20:14:45,089 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T20:14:45,088 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-26T20:14:45,090 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-26T20:14:45,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure c156031df6143a0a099724d68db39599, server=76919af20f29,44765,1732652055399:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 76919af20f29,38853,1732652055556, splitWal=true, meta=true exec-time=4.8250 sec 2024-11-26T20:14:45,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5070f0e2d0547381b8d60e94c76cdaff, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b8a0db8a2157574a65d6d70ba9f15e1, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a7108f1289f21074490c53bbef06d630, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1e22e09e90e1a05e430419c43da9c80c, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,094 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e78554385cc62e1f4a220f96a9d9207d, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5632c367c5d9113b557fd742bcb2b62f, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34cb9834732a35daa158cb86b8b3b142, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dde94c133aedae28d634a201ddb741f7, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87979a5e316fc73bab2f01285393a426, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ca44b38774b13852ff96928138ba075, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f5d1c8271bbf4e7a65c1cc98ef8dc026, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5249ac65133ea81926c57db86f9b90e, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a569d698819974eb6636a41f610467c, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,095 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=340a1869e06658f75d054f357d9a5c66, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=78cd4b9200f83099c4095aeeffca9b21, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d1c0c8c5fd0d7b81fc270155b15f8b86, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=726bb7d945255f0a9e4ebbb78c65b7d5, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba6d284e3fe19dfa6e13c7648e255291, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c156031df6143a0a099724d68db39599, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c0c6f3b0d7be2eee5a98bcf417b7ba9e, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=39562e7939506cf96cc70158f24d8ea2, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a9d36c52dee7a92acb00483dfe6052, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0880a54a28990b0c067cdb9447a1593b, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9fa8bd8119f33b0d6fffe6254be9f4b, regionState=OPENING, lastHost=76919af20f29,38853,1732652055556, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6500def2f64bee0d698a3c7fb6413e30, regionState=OPEN, lastHost=76919af20f29,44765,1732652055399, regionLocation=76919af20f29,44765,1732652055399, openSeqNum=2 2024-11-26T20:14:45,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=965d8bc3f7f9b8218376c29c89db6db3, regionState=OPEN, lastHost=76919af20f29,43267,1732652055155, regionLocation=76919af20f29,43267,1732652055155, openSeqNum=2 2024-11-26T20:14:45,097 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-26T20:14:45,097 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732652145097 2024-11-26T20:14:45,097 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732652205097 2024-11-26T20:14:45,097 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-26T20:14:45,113 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 379), OpenFileDescriptor=600 (was 617), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1323 (was 1352), ProcessCount=11 (was 11), AvailableMemoryMB=4235 (was 294) - AvailableMemoryMB LEAK? - 2024-11-26T20:14:45,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T20:14:45,113 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:14:45,113 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:14:45,113 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:45,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:45,114 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T20:14:45,114 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T20:14:45,114 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1570076000, stopped=false 2024-11-26T20:14:45,114 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=76919af20f29,34091,1732652053862 2024-11-26T20:14:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:14:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:14:45,124 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T20:14:45,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:45,125 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T20:14:45,125 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:14:45,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:45,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:14:45,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:14:45,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T20:14:45,125 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '76919af20f29,43267,1732652055155' ***** 2024-11-26T20:14:45,125 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T20:14:45,125 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '76919af20f29,44765,1732652055399' ***** 2024-11-26T20:14:45,125 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T20:14:45,126 INFO [RS:1;76919af20f29:44765 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:14:45,126 INFO [RS:0;76919af20f29:43267 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T20:14:45,126 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:14:45,126 INFO [RS:1;76919af20f29:44765 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T20:14:45,126 INFO [RS:0;76919af20f29:43267 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T20:14:45,126 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T20:14:45,126 INFO [RS:1;76919af20f29:44765 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T20:14:45,126 INFO [RS:0;76919af20f29:43267 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T20:14:45,126 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for 10a9d36c52dee7a92acb00483dfe6052 2024-11-26T20:14:45,126 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 6500def2f64bee0d698a3c7fb6413e30 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for 965d8bc3f7f9b8218376c29c89db6db3 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for 4b8a0db8a2157574a65d6d70ba9f15e1 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for 2ca44b38774b13852ff96928138ba075 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for 78cd4b9200f83099c4095aeeffca9b21 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for dde94c133aedae28d634a201ddb741f7 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for 39562e7939506cf96cc70158f24d8ea2 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for e78554385cc62e1f4a220f96a9d9207d 2024-11-26T20:14:45,127 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 10a9d36c52dee7a92acb00483dfe6052, disabling compactions & flushes 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for a7108f1289f21074490c53bbef06d630 2024-11-26T20:14:45,127 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:45,127 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for ba6d284e3fe19dfa6e13c7648e255291 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for 0a569d698819974eb6636a41f610467c 2024-11-26T20:14:45,127 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. after waiting 0 ms 2024-11-26T20:14:45,127 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(959): stopping server 76919af20f29,43267,1732652055155 2024-11-26T20:14:45,127 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:14:45,127 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 34cb9834732a35daa158cb86b8b3b142 2024-11-26T20:14:45,127 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 5070f0e2d0547381b8d60e94c76cdaff 2024-11-26T20:14:45,127 INFO [RS:0;76919af20f29:43267 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;76919af20f29:43267. 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 340a1869e06658f75d054f357d9a5c66 2024-11-26T20:14:45,128 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6500def2f64bee0d698a3c7fb6413e30, disabling compactions & flushes 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 1e22e09e90e1a05e430419c43da9c80c 2024-11-26T20:14:45,128 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:45,128 DEBUG [RS:0;76919af20f29:43267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 726bb7d945255f0a9e4ebbb78c65b7d5 2024-11-26T20:14:45,128 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for c156031df6143a0a099724d68db39599 2024-11-26T20:14:45,128 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. after waiting 0 ms 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 87979a5e316fc73bab2f01285393a426 2024-11-26T20:14:45,128 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 5632c367c5d9113b557fd742bcb2b62f 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for c0c6f3b0d7be2eee5a98bcf417b7ba9e 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:45,128 DEBUG [RS:0;76919af20f29:43267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(3091): Received CLOSE for 0880a54a28990b0c067cdb9447a1593b 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(959): stopping server 76919af20f29,44765,1732652055399 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;76919af20f29:44765. 2024-11-26T20:14:45,128 DEBUG [RS:1;76919af20f29:44765 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T20:14:45,128 DEBUG [RS:1;76919af20f29:44765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:45,128 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-26T20:14:45,129 INFO [RS:0;76919af20f29:43267 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T20:14:45,129 INFO [RS:0;76919af20f29:43267 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T20:14:45,129 INFO [RS:0;76919af20f29:43267 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T20:14:45,129 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T20:14:45,129 DEBUG [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1351): Waiting on 0880a54a28990b0c067cdb9447a1593b, 1e22e09e90e1a05e430419c43da9c80c, 340a1869e06658f75d054f357d9a5c66, 34cb9834732a35daa158cb86b8b3b142, 5070f0e2d0547381b8d60e94c76cdaff, 5632c367c5d9113b557fd742bcb2b62f, 6500def2f64bee0d698a3c7fb6413e30, 726bb7d945255f0a9e4ebbb78c65b7d5, 87979a5e316fc73bab2f01285393a426, c0c6f3b0d7be2eee5a98bcf417b7ba9e, c156031df6143a0a099724d68db39599, c5249ac65133ea81926c57db86f9b90e, f9fa8bd8119f33b0d6fffe6254be9f4b 2024-11-26T20:14:45,129 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-26T20:14:45,129 DEBUG [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1351): Waiting on 0a569d698819974eb6636a41f610467c, 10a9d36c52dee7a92acb00483dfe6052, 1588230740, 2ca44b38774b13852ff96928138ba075, 39562e7939506cf96cc70158f24d8ea2, 4b8a0db8a2157574a65d6d70ba9f15e1, 78cd4b9200f83099c4095aeeffca9b21, 965d8bc3f7f9b8218376c29c89db6db3, a7108f1289f21074490c53bbef06d630, ba6d284e3fe19dfa6e13c7648e255291, d1c0c8c5fd0d7b81fc270155b15f8b86, dde94c133aedae28d634a201ddb741f7, e78554385cc62e1f4a220f96a9d9207d, f5d1c8271bbf4e7a65c1cc98ef8dc026 2024-11-26T20:14:45,130 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T20:14:45,130 INFO [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T20:14:45,130 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T20:14:45,130 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T20:14:45,130 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T20:14:45,130 INFO [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-26T20:14:45,137 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/10a9d36c52dee7a92acb00483dfe6052/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,138 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/6500def2f64bee0d698a3c7fb6413e30/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,140 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:45,140 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 10a9d36c52dee7a92acb00483dfe6052: Waiting for close lock at 1732652085127Running coprocessor pre-close hooks at 1732652085127Disabling compacts and flushes for region at 1732652085127Disabling writes for close at 1732652085127Writing region close event to WAL at 1732652085128 (+1 ms)Running coprocessor post-close hooks at 1732652085138 (+10 ms)Closed at 1732652085140 (+2 ms) 2024-11-26T20:14:45,140 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:45,140 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6500def2f64bee0d698a3c7fb6413e30: Waiting for close lock at 1732652085127Running coprocessor pre-close hooks at 1732652085128 (+1 ms)Disabling compacts and flushes for region at 1732652085128Disabling writes for close at 1732652085128Writing region close event to WAL at 1732652085129 (+1 ms)Running coprocessor post-close hooks at 1732652085140 (+11 ms)Closed at 1732652085140 2024-11-26T20:14:45,140 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732652061178.10a9d36c52dee7a92acb00483dfe6052. 2024-11-26T20:14:45,140 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732652061178.6500def2f64bee0d698a3c7fb6413e30. 2024-11-26T20:14:45,141 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f9fa8bd8119f33b0d6fffe6254be9f4b, disabling compactions & flushes 2024-11-26T20:14:45,141 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 965d8bc3f7f9b8218376c29c89db6db3, disabling compactions & flushes 2024-11-26T20:14:45,141 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:45,141 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:45,141 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:45,141 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:45,141 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. after waiting 0 ms 2024-11-26T20:14:45,141 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. after waiting 0 ms 2024-11-26T20:14:45,141 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:45,141 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:45,152 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/info/6b07a6314c4d418e980a2790fd133f79 is 138, key is test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630./info:regioninfo/1732652082889/Put/seqid=0 2024-11-26T20:14:45,161 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/965d8bc3f7f9b8218376c29c89db6db3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,161 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f9fa8bd8119f33b0d6fffe6254be9f4b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,162 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:45,162 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 965d8bc3f7f9b8218376c29c89db6db3: Waiting for close lock at 1732652085141Running coprocessor pre-close hooks at 1732652085141Disabling compacts and flushes for region at 1732652085141Disabling writes for close at 1732652085141Writing region close event to WAL at 1732652085145 (+4 ms)Running coprocessor post-close hooks at 1732652085162 (+17 ms)Closed at 1732652085162 2024-11-26T20:14:45,162 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732652061178.965d8bc3f7f9b8218376c29c89db6db3. 2024-11-26T20:14:45,162 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:45,162 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f9fa8bd8119f33b0d6fffe6254be9f4b: Waiting for close lock at 1732652085141Running coprocessor pre-close hooks at 1732652085141Disabling compacts and flushes for region at 1732652085141Disabling writes for close at 1732652085141Writing region close event to WAL at 1732652085141Running coprocessor post-close hooks at 1732652085162 (+21 ms)Closed at 1732652085162 2024-11-26T20:14:45,162 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732652061178.f9fa8bd8119f33b0d6fffe6254be9f4b. 2024-11-26T20:14:45,162 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4b8a0db8a2157574a65d6d70ba9f15e1, disabling compactions & flushes 2024-11-26T20:14:45,162 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:45,162 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:45,162 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34cb9834732a35daa158cb86b8b3b142, disabling compactions & flushes 2024-11-26T20:14:45,163 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. after waiting 0 ms 2024-11-26T20:14:45,163 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:45,163 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:45,163 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:45,163 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. after waiting 0 ms 2024-11-26T20:14:45,163 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:45,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741872_1048 (size=8195) 2024-11-26T20:14:45,178 INFO [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/info/6b07a6314c4d418e980a2790fd133f79 2024-11-26T20:14:45,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741872_1048 (size=8195) 2024-11-26T20:14:45,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741872_1048 (size=8195) 2024-11-26T20:14:45,185 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/4b8a0db8a2157574a65d6d70ba9f15e1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,187 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:45,188 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4b8a0db8a2157574a65d6d70ba9f15e1: Waiting for close lock at 1732652085162Running coprocessor pre-close hooks at 1732652085162Disabling compacts and flushes for region at 1732652085162Disabling writes for close at 1732652085163 (+1 ms)Writing region close event to WAL at 1732652085164 (+1 ms)Running coprocessor post-close hooks at 1732652085187 (+23 ms)Closed at 1732652085187 2024-11-26T20:14:45,188 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/34cb9834732a35daa158cb86b8b3b142/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,188 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732652061178.4b8a0db8a2157574a65d6d70ba9f15e1. 2024-11-26T20:14:45,188 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2ca44b38774b13852ff96928138ba075, disabling compactions & flushes 2024-11-26T20:14:45,189 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. after waiting 0 ms 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:45,189 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34cb9834732a35daa158cb86b8b3b142: Waiting for close lock at 1732652085162Running coprocessor pre-close hooks at 1732652085162Disabling compacts and flushes for region at 1732652085162Disabling writes for close at 1732652085163 (+1 ms)Writing region close event to WAL at 1732652085168 (+5 ms)Running coprocessor post-close hooks at 1732652085189 (+21 ms)Closed at 1732652085189 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732652061178.34cb9834732a35daa158cb86b8b3b142. 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5070f0e2d0547381b8d60e94c76cdaff, disabling compactions & flushes 2024-11-26T20:14:45,189 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. after waiting 0 ms 2024-11-26T20:14:45,189 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:45,190 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/.tmp/info/6b07a6314c4d418e980a2790fd133f79 as hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/info/6b07a6314c4d418e980a2790fd133f79 2024-11-26T20:14:45,198 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5070f0e2d0547381b8d60e94c76cdaff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,199 INFO [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/info/6b07a6314c4d418e980a2790fd133f79, entries=24, sequenceid=71, filesize=8.0 K 2024-11-26T20:14:45,199 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:45,199 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5070f0e2d0547381b8d60e94c76cdaff: Waiting for close lock at 1732652085189Running coprocessor pre-close hooks at 1732652085189Disabling compacts and flushes for region at 1732652085189Disabling writes for close at 1732652085189Writing region close event to WAL at 1732652085193 (+4 ms)Running coprocessor post-close hooks at 1732652085199 (+6 ms)Closed at 1732652085199 2024-11-26T20:14:45,200 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732652061178.5070f0e2d0547381b8d60e94c76cdaff. 2024-11-26T20:14:45,200 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 340a1869e06658f75d054f357d9a5c66, disabling compactions & flushes 2024-11-26T20:14:45,200 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:45,200 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:45,200 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. after waiting 0 ms 2024-11-26T20:14:45,200 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:45,201 INFO [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=71, compaction requested=false 2024-11-26T20:14:45,202 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/2ca44b38774b13852ff96928138ba075/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,203 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:45,203 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2ca44b38774b13852ff96928138ba075: Waiting for close lock at 1732652085188Running coprocessor pre-close hooks at 1732652085188Disabling compacts and flushes for region at 1732652085188Disabling writes for close at 1732652085189 (+1 ms)Writing region close event to WAL at 1732652085193 (+4 ms)Running coprocessor post-close hooks at 1732652085203 (+10 ms)Closed at 1732652085203 2024-11-26T20:14:45,203 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732652061178.2ca44b38774b13852ff96928138ba075. 2024-11-26T20:14:45,204 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 78cd4b9200f83099c4095aeeffca9b21, disabling compactions & flushes 2024-11-26T20:14:45,204 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:45,204 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:45,204 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. after waiting 0 ms 2024-11-26T20:14:45,204 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:45,210 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/340a1869e06658f75d054f357d9a5c66/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,212 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:45,212 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 340a1869e06658f75d054f357d9a5c66: Waiting for close lock at 1732652085200Running coprocessor pre-close hooks at 1732652085200Disabling compacts and flushes for region at 1732652085200Disabling writes for close at 1732652085200Writing region close event to WAL at 1732652085200Running coprocessor post-close hooks at 1732652085212 (+12 ms)Closed at 1732652085212 2024-11-26T20:14:45,212 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732652061178.340a1869e06658f75d054f357d9a5c66. 2024-11-26T20:14:45,212 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1e22e09e90e1a05e430419c43da9c80c, disabling compactions & flushes 2024-11-26T20:14:45,212 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:45,212 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:45,212 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. after waiting 0 ms 2024-11-26T20:14:45,212 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:45,213 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/78cd4b9200f83099c4095aeeffca9b21/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,214 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:45,215 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 78cd4b9200f83099c4095aeeffca9b21: Waiting for close lock at 1732652085204Running coprocessor pre-close hooks at 1732652085204Disabling compacts and flushes for region at 1732652085204Disabling writes for close at 1732652085204Writing region close event to WAL at 1732652085204Running coprocessor post-close hooks at 1732652085214 (+10 ms)Closed at 1732652085214 2024-11-26T20:14:45,215 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732652061178.78cd4b9200f83099c4095aeeffca9b21. 2024-11-26T20:14:45,215 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f5d1c8271bbf4e7a65c1cc98ef8dc026, disabling compactions & flushes 2024-11-26T20:14:45,215 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:45,215 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:45,215 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. after waiting 0 ms 2024-11-26T20:14:45,215 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:45,217 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-26T20:14:45,218 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T20:14:45,218 INFO [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T20:14:45,218 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732652085129Running coprocessor pre-close hooks at 1732652085129Disabling compacts and flushes for region at 1732652085129Disabling writes for close at 1732652085130 (+1 ms)Obtaining lock to block concurrent updates at 1732652085130Preparing flush snapshotting stores in 1588230740 at 1732652085130Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732652085136 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732652085137 (+1 ms)Flushing 1588230740/info: creating writer at 1732652085138 (+1 ms)Flushing 1588230740/info: appending metadata at 1732652085151 (+13 ms)Flushing 1588230740/info: closing flushed file at 1732652085151Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63d52377: reopening flushed file at 1732652085189 (+38 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 70ms, sequenceid=71, compaction requested=false at 1732652085201 (+12 ms)Writing region close event to WAL at 1732652085205 (+4 ms)Running coprocessor post-close hooks at 1732652085218 (+13 ms)Closed at 1732652085218 2024-11-26T20:14:45,218 DEBUG [RS_CLOSE_META-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T20:14:45,224 INFO [regionserver/76919af20f29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:14:45,224 INFO [regionserver/76919af20f29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:14:45,233 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/1e22e09e90e1a05e430419c43da9c80c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,233 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:45,234 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1e22e09e90e1a05e430419c43da9c80c: Waiting for close lock at 1732652085212Running coprocessor pre-close hooks at 1732652085212Disabling compacts and flushes for region at 1732652085212Disabling writes for close at 1732652085212Writing region close event to WAL at 1732652085227 (+15 ms)Running coprocessor post-close hooks at 1732652085233 (+6 ms)Closed at 1732652085233 2024-11-26T20:14:45,234 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732652061178.1e22e09e90e1a05e430419c43da9c80c. 2024-11-26T20:14:45,234 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 726bb7d945255f0a9e4ebbb78c65b7d5, disabling compactions & flushes 2024-11-26T20:14:45,234 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:45,234 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:45,234 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. after waiting 0 ms 2024-11-26T20:14:45,234 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:45,238 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/f5d1c8271bbf4e7a65c1cc98ef8dc026/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,240 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:45,240 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f5d1c8271bbf4e7a65c1cc98ef8dc026: Waiting for close lock at 1732652085215Running coprocessor pre-close hooks at 1732652085215Disabling compacts and flushes for region at 1732652085215Disabling writes for close at 1732652085215Writing region close event to WAL at 1732652085229 (+14 ms)Running coprocessor post-close hooks at 1732652085240 (+11 ms)Closed at 1732652085240 2024-11-26T20:14:45,240 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732652061178.f5d1c8271bbf4e7a65c1cc98ef8dc026. 2024-11-26T20:14:45,240 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dde94c133aedae28d634a201ddb741f7, disabling compactions & flushes 2024-11-26T20:14:45,240 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:45,240 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:45,240 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. after waiting 0 ms 2024-11-26T20:14:45,241 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:45,247 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/726bb7d945255f0a9e4ebbb78c65b7d5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,248 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:45,249 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 726bb7d945255f0a9e4ebbb78c65b7d5: Waiting for close lock at 1732652085234Running coprocessor pre-close hooks at 1732652085234Disabling compacts and flushes for region at 1732652085234Disabling writes for close at 1732652085234Writing region close event to WAL at 1732652085236 (+2 ms)Running coprocessor post-close hooks at 1732652085248 (+12 ms)Closed at 1732652085248 2024-11-26T20:14:45,249 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732652061178.726bb7d945255f0a9e4ebbb78c65b7d5. 2024-11-26T20:14:45,249 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c156031df6143a0a099724d68db39599, disabling compactions & flushes 2024-11-26T20:14:45,249 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:45,249 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:45,249 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732652061178.c156031df6143a0a099724d68db39599. after waiting 0 ms 2024-11-26T20:14:45,249 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:45,253 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/dde94c133aedae28d634a201ddb741f7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,254 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:45,254 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dde94c133aedae28d634a201ddb741f7: Waiting for close lock at 1732652085240Running coprocessor pre-close hooks at 1732652085240Disabling compacts and flushes for region at 1732652085240Disabling writes for close at 1732652085240Writing region close event to WAL at 1732652085244 (+4 ms)Running coprocessor post-close hooks at 1732652085254 (+10 ms)Closed at 1732652085254 2024-11-26T20:14:45,255 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732652061178.dde94c133aedae28d634a201ddb741f7. 2024-11-26T20:14:45,255 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 39562e7939506cf96cc70158f24d8ea2, disabling compactions & flushes 2024-11-26T20:14:45,255 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:45,255 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:45,255 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. after waiting 0 ms 2024-11-26T20:14:45,255 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:45,262 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c156031df6143a0a099724d68db39599/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,263 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:45,263 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c156031df6143a0a099724d68db39599: Waiting for close lock at 1732652085249Running coprocessor pre-close hooks at 1732652085249Disabling compacts and flushes for region at 1732652085249Disabling writes for close at 1732652085249Writing region close event to WAL at 1732652085250 (+1 ms)Running coprocessor post-close hooks at 1732652085263 (+13 ms)Closed at 1732652085263 2024-11-26T20:14:45,264 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732652061178.c156031df6143a0a099724d68db39599. 2024-11-26T20:14:45,264 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 87979a5e316fc73bab2f01285393a426, disabling compactions & flushes 2024-11-26T20:14:45,264 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:45,264 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:45,264 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. after waiting 0 ms 2024-11-26T20:14:45,264 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:45,264 INFO [regionserver/76919af20f29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T20:14:45,264 INFO [regionserver/76919af20f29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T20:14:45,277 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/39562e7939506cf96cc70158f24d8ea2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,277 INFO [regionserver/76919af20f29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T20:14:45,278 INFO [regionserver/76919af20f29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T20:14:45,282 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:45,282 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 39562e7939506cf96cc70158f24d8ea2: Waiting for close lock at 1732652085255Running coprocessor pre-close hooks at 1732652085255Disabling compacts and flushes for region at 1732652085255Disabling writes for close at 1732652085255Writing region close event to WAL at 1732652085260 (+5 ms)Running coprocessor post-close hooks at 1732652085281 (+21 ms)Closed at 1732652085281 2024-11-26T20:14:45,282 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732652061178.39562e7939506cf96cc70158f24d8ea2. 2024-11-26T20:14:45,282 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e78554385cc62e1f4a220f96a9d9207d, disabling compactions & flushes 2024-11-26T20:14:45,282 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:45,282 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:45,282 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. after waiting 0 ms 2024-11-26T20:14:45,282 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:45,286 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/87979a5e316fc73bab2f01285393a426/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,287 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:45,287 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 87979a5e316fc73bab2f01285393a426: Waiting for close lock at 1732652085264Running coprocessor pre-close hooks at 1732652085264Disabling compacts and flushes for region at 1732652085264Disabling writes for close at 1732652085264Writing region close event to WAL at 1732652085268 (+4 ms)Running coprocessor post-close hooks at 1732652085287 (+19 ms)Closed at 1732652085287 2024-11-26T20:14:45,287 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732652061178.87979a5e316fc73bab2f01285393a426. 2024-11-26T20:14:45,287 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5632c367c5d9113b557fd742bcb2b62f, disabling compactions & flushes 2024-11-26T20:14:45,287 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:45,287 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:45,287 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. after waiting 0 ms 2024-11-26T20:14:45,287 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:45,308 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/e78554385cc62e1f4a220f96a9d9207d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,309 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:45,309 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e78554385cc62e1f4a220f96a9d9207d: Waiting for close lock at 1732652085282Running coprocessor pre-close hooks at 1732652085282Disabling compacts and flushes for region at 1732652085282Disabling writes for close at 1732652085282Writing region close event to WAL at 1732652085288 (+6 ms)Running coprocessor post-close hooks at 1732652085309 (+21 ms)Closed at 1732652085309 2024-11-26T20:14:45,310 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732652061178.e78554385cc62e1f4a220f96a9d9207d. 2024-11-26T20:14:45,310 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a7108f1289f21074490c53bbef06d630, disabling compactions & flushes 2024-11-26T20:14:45,310 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:45,310 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:45,310 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. after waiting 0 ms 2024-11-26T20:14:45,310 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:45,316 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/5632c367c5d9113b557fd742bcb2b62f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,317 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:45,318 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5632c367c5d9113b557fd742bcb2b62f: Waiting for close lock at 1732652085287Running coprocessor pre-close hooks at 1732652085287Disabling compacts and flushes for region at 1732652085287Disabling writes for close at 1732652085287Writing region close event to WAL at 1732652085291 (+4 ms)Running coprocessor post-close hooks at 1732652085317 (+26 ms)Closed at 1732652085317 2024-11-26T20:14:45,318 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732652061178.5632c367c5d9113b557fd742bcb2b62f. 2024-11-26T20:14:45,318 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c0c6f3b0d7be2eee5a98bcf417b7ba9e, disabling compactions & flushes 2024-11-26T20:14:45,318 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:45,318 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:45,318 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. after waiting 0 ms 2024-11-26T20:14:45,318 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:45,327 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/a7108f1289f21074490c53bbef06d630/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,327 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c0c6f3b0d7be2eee5a98bcf417b7ba9e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,328 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:45,328 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a7108f1289f21074490c53bbef06d630: Waiting for close lock at 1732652085310Running coprocessor pre-close hooks at 1732652085310Disabling compacts and flushes for region at 1732652085310Disabling writes for close at 1732652085310Writing region close event to WAL at 1732652085316 (+6 ms)Running coprocessor post-close hooks at 1732652085328 (+12 ms)Closed at 1732652085328 2024-11-26T20:14:45,328 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732652061178.a7108f1289f21074490c53bbef06d630. 2024-11-26T20:14:45,328 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d1c0c8c5fd0d7b81fc270155b15f8b86, disabling compactions & flushes 2024-11-26T20:14:45,329 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:45,329 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:45,329 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. after waiting 0 ms 2024-11-26T20:14:45,329 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:45,329 DEBUG [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1351): Waiting on 0880a54a28990b0c067cdb9447a1593b, c0c6f3b0d7be2eee5a98bcf417b7ba9e, c5249ac65133ea81926c57db86f9b90e 2024-11-26T20:14:45,329 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:45,329 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c0c6f3b0d7be2eee5a98bcf417b7ba9e: Waiting for close lock at 1732652085318Running coprocessor pre-close hooks at 1732652085318Disabling compacts and flushes for region at 1732652085318Disabling writes for close at 1732652085318Writing region close event to WAL at 1732652085318Running coprocessor post-close hooks at 1732652085329 (+11 ms)Closed at 1732652085329 2024-11-26T20:14:45,330 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732652061178.c0c6f3b0d7be2eee5a98bcf417b7ba9e. 2024-11-26T20:14:45,330 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c5249ac65133ea81926c57db86f9b90e, disabling compactions & flushes 2024-11-26T20:14:45,330 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:45,330 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:45,330 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. after waiting 0 ms 2024-11-26T20:14:45,330 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:45,332 DEBUG [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1351): Waiting on 0a569d698819974eb6636a41f610467c, ba6d284e3fe19dfa6e13c7648e255291, d1c0c8c5fd0d7b81fc270155b15f8b86 2024-11-26T20:14:45,345 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/d1c0c8c5fd0d7b81fc270155b15f8b86/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,345 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/c5249ac65133ea81926c57db86f9b90e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,346 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:45,346 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c5249ac65133ea81926c57db86f9b90e: Waiting for close lock at 1732652085330Running coprocessor pre-close hooks at 1732652085330Disabling compacts and flushes for region at 1732652085330Disabling writes for close at 1732652085330Writing region close event to WAL at 1732652085336 (+6 ms)Running coprocessor post-close hooks at 1732652085346 (+10 ms)Closed at 1732652085346 2024-11-26T20:14:45,346 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:45,346 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d1c0c8c5fd0d7b81fc270155b15f8b86: Waiting for close lock at 1732652085328Running coprocessor pre-close hooks at 1732652085328Disabling compacts and flushes for region at 1732652085328Disabling writes for close at 1732652085329 (+1 ms)Writing region close event to WAL at 1732652085329Running coprocessor post-close hooks at 1732652085346 (+17 ms)Closed at 1732652085346 2024-11-26T20:14:45,346 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732652061178.c5249ac65133ea81926c57db86f9b90e. 2024-11-26T20:14:45,346 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732652061178.d1c0c8c5fd0d7b81fc270155b15f8b86. 2024-11-26T20:14:45,346 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0880a54a28990b0c067cdb9447a1593b, disabling compactions & flushes 2024-11-26T20:14:45,346 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ba6d284e3fe19dfa6e13c7648e255291, disabling compactions & flushes 2024-11-26T20:14:45,346 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:45,347 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:45,347 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:45,347 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:45,347 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. after waiting 0 ms 2024-11-26T20:14:45,347 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. after waiting 0 ms 2024-11-26T20:14:45,347 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:45,347 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:45,357 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/0880a54a28990b0c067cdb9447a1593b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,358 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:45,358 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0880a54a28990b0c067cdb9447a1593b: Waiting for close lock at 1732652085346Running coprocessor pre-close hooks at 1732652085346Disabling compacts and flushes for region at 1732652085346Disabling writes for close at 1732652085347 (+1 ms)Writing region close event to WAL at 1732652085347Running coprocessor post-close hooks at 1732652085358 (+11 ms)Closed at 1732652085358 2024-11-26T20:14:45,358 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732652061178.0880a54a28990b0c067cdb9447a1593b. 2024-11-26T20:14:45,360 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/ba6d284e3fe19dfa6e13c7648e255291/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,361 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:45,361 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ba6d284e3fe19dfa6e13c7648e255291: Waiting for close lock at 1732652085346Running coprocessor pre-close hooks at 1732652085346Disabling compacts and flushes for region at 1732652085346Disabling writes for close at 1732652085347 (+1 ms)Writing region close event to WAL at 1732652085347Running coprocessor post-close hooks at 1732652085361 (+14 ms)Closed at 1732652085361 2024-11-26T20:14:45,361 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732652061178.ba6d284e3fe19dfa6e13c7648e255291. 2024-11-26T20:14:45,362 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0a569d698819974eb6636a41f610467c, disabling compactions & flushes 2024-11-26T20:14:45,362 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:45,362 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:45,362 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. after waiting 0 ms 2024-11-26T20:14:45,362 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:45,382 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/data/default/test/0a569d698819974eb6636a41f610467c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-26T20:14:45,383 INFO [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:45,383 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0a569d698819974eb6636a41f610467c: Waiting for close lock at 1732652085362Running coprocessor pre-close hooks at 1732652085362Disabling compacts and flushes for region at 1732652085362Disabling writes for close at 1732652085362Writing region close event to WAL at 1732652085364 (+2 ms)Running coprocessor post-close hooks at 1732652085383 (+19 ms)Closed at 1732652085383 2024-11-26T20:14:45,383 DEBUG [RS_CLOSE_REGION-regionserver/76919af20f29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732652061178.0a569d698819974eb6636a41f610467c. 2024-11-26T20:14:45,529 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(976): stopping server 76919af20f29,44765,1732652055399; all regions closed. 2024-11-26T20:14:45,532 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(976): stopping server 76919af20f29,43267,1732652055155; all regions closed. 2024-11-26T20:14:45,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741834_1010 (size=7064) 2024-11-26T20:14:45,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741834_1010 (size=7064) 2024-11-26T20:14:45,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741834_1010 (size=7064) 2024-11-26T20:14:45,535 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/WALs/76919af20f29,44765,1732652055399/76919af20f29%2C44765%2C1732652055399.1732652059453 not finished, retry = 0 2024-11-26T20:14:45,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741868_1044 (size=4204) 2024-11-26T20:14:45,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741868_1044 (size=4204) 2024-11-26T20:14:45,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741868_1044 (size=4204) 2024-11-26T20:14:45,545 DEBUG [RS:0;76919af20f29:43267 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs 2024-11-26T20:14:45,545 INFO [RS:0;76919af20f29:43267 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 76919af20f29%2C43267%2C1732652055155.meta:.meta(num 1732652081336) 2024-11-26T20:14:45,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741835_1011 (size=7074) 2024-11-26T20:14:45,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741835_1011 (size=7074) 2024-11-26T20:14:45,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741835_1011 (size=7074) 2024-11-26T20:14:45,555 DEBUG [RS:0;76919af20f29:43267 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs 2024-11-26T20:14:45,555 INFO [RS:0;76919af20f29:43267 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 76919af20f29%2C43267%2C1732652055155:(num 1732652059481) 2024-11-26T20:14:45,555 DEBUG [RS:0;76919af20f29:43267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:45,555 INFO [RS:0;76919af20f29:43267 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:14:45,556 INFO [RS:0;76919af20f29:43267 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:14:45,556 INFO [RS:0;76919af20f29:43267 {}] hbase.ChoreService(370): Chore service for: regionserver/76919af20f29:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T20:14:45,557 INFO [RS:0;76919af20f29:43267 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:14:45,557 INFO [regionserver/76919af20f29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:14:45,557 INFO [RS:0;76919af20f29:43267 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43267 2024-11-26T20:14:45,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/76919af20f29,43267,1732652055155 2024-11-26T20:14:45,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:14:45,634 INFO [RS:0;76919af20f29:43267 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:14:45,639 DEBUG [RS:1;76919af20f29:44765 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/oldWALs 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 76919af20f29%2C44765%2C1732652055399:(num 1732652059453) 2024-11-26T20:14:45,639 DEBUG [RS:1;76919af20f29:44765 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] hbase.ChoreService(370): Chore service for: regionserver/76919af20f29:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:14:45,639 INFO [RS:1;76919af20f29:44765 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44765 2024-11-26T20:14:45,640 INFO [regionserver/76919af20f29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:14:45,644 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [76919af20f29,43267,1732652055155] 2024-11-26T20:14:45,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T20:14:45,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/76919af20f29,44765,1732652055399 2024-11-26T20:14:45,655 INFO [RS:1;76919af20f29:44765 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:14:45,665 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/76919af20f29,43267,1732652055155 already deleted, retry=false 2024-11-26T20:14:45,665 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 76919af20f29,43267,1732652055155 expired; onlineServers=1 2024-11-26T20:14:45,675 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [76919af20f29,44765,1732652055399] 2024-11-26T20:14:45,686 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/76919af20f29,44765,1732652055399 already deleted, retry=false 2024-11-26T20:14:45,686 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 76919af20f29,44765,1732652055399 expired; onlineServers=0 2024-11-26T20:14:45,686 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '76919af20f29,34091,1732652053862' ***** 2024-11-26T20:14:45,686 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T20:14:45,686 INFO [M:0;76919af20f29:34091 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T20:14:45,687 INFO [M:0;76919af20f29:34091 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T20:14:45,687 DEBUG [M:0;76919af20f29:34091 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T20:14:45,687 DEBUG [M:0;76919af20f29:34091 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T20:14:45,687 DEBUG [master/76919af20f29:0:becomeActiveMaster-HFileCleaner.small.0-1732652058648 {}] cleaner.HFileCleaner(306): Exit Thread[master/76919af20f29:0:becomeActiveMaster-HFileCleaner.small.0-1732652058648,5,FailOnTimeoutGroup] 2024-11-26T20:14:45,687 DEBUG [master/76919af20f29:0:becomeActiveMaster-HFileCleaner.large.0-1732652058617 {}] cleaner.HFileCleaner(306): Exit Thread[master/76919af20f29:0:becomeActiveMaster-HFileCleaner.large.0-1732652058617,5,FailOnTimeoutGroup] 2024-11-26T20:14:45,687 INFO [M:0;76919af20f29:34091 {}] hbase.ChoreService(370): Chore service for: master/76919af20f29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T20:14:45,688 INFO [M:0;76919af20f29:34091 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T20:14:45,688 DEBUG [M:0;76919af20f29:34091 {}] master.HMaster(1795): Stopping service threads 2024-11-26T20:14:45,688 INFO [M:0;76919af20f29:34091 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T20:14:45,688 INFO [M:0;76919af20f29:34091 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T20:14:45,688 INFO [M:0;76919af20f29:34091 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T20:14:45,688 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T20:14:45,687 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T20:14:45,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T20:14:45,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T20:14:45,707 DEBUG [M:0;76919af20f29:34091 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-26T20:14:45,707 DEBUG [M:0;76919af20f29:34091 {}] master.ActiveMasterManager(353): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-26T20:14:45,710 INFO [M:0;76919af20f29:34091 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/.lastflushedseqids 2024-11-26T20:14:45,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:14:45,745 INFO [RS:0;76919af20f29:43267 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:14:45,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43267-0x10178c82cdf0001, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:14:45,745 INFO [RS:0;76919af20f29:43267 {}] regionserver.HRegionServer(1031): Exiting; stopping=76919af20f29,43267,1732652055155; zookeeper connection closed. 2024-11-26T20:14:45,750 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a72795f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a72795f 2024-11-26T20:14:45,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741873_1049 (size=1832) 2024-11-26T20:14:45,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741873_1049 (size=1832) 2024-11-26T20:14:45,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741873_1049 (size=1832) 2024-11-26T20:14:45,763 INFO [M:0;76919af20f29:34091 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T20:14:45,763 INFO [M:0;76919af20f29:34091 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T20:14:45,763 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T20:14:45,763 INFO [M:0;76919af20f29:34091 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:14:45,763 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:14:45,763 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T20:14:45,763 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:14:45,763 INFO [M:0;76919af20f29:34091 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=257.93 KB heapSize=313.17 KB 2024-11-26T20:14:45,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:14:45,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44765-0x10178c82cdf0002, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:14:45,780 INFO [RS:1;76919af20f29:44765 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:14:45,780 INFO [RS:1;76919af20f29:44765 {}] regionserver.HRegionServer(1031): Exiting; stopping=76919af20f29,44765,1732652055399; zookeeper connection closed. 2024-11-26T20:14:45,784 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5c87f4ee {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5c87f4ee 2024-11-26T20:14:45,785 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-26T20:14:45,795 DEBUG [M:0;76919af20f29:34091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/77cd35eadd3a45c083b520fb6c25d1f8 is 82, key is hbase:meta,,1/info:regioninfo/1732652082471/Put/seqid=0 2024-11-26T20:14:45,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741874_1050 (size=6071) 2024-11-26T20:14:45,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741874_1050 (size=6071) 2024-11-26T20:14:45,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741874_1050 (size=6071) 2024-11-26T20:14:45,832 INFO [M:0;76919af20f29:34091 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/77cd35eadd3a45c083b520fb6c25d1f8 2024-11-26T20:14:45,863 DEBUG [M:0;76919af20f29:34091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f92770bc51074bb399d73e69f41020b2 is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732652085090/Put/seqid=0 2024-11-26T20:14:45,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741875_1051 (size=12873) 2024-11-26T20:14:45,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741875_1051 (size=12873) 2024-11-26T20:14:45,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741875_1051 (size=12873) 2024-11-26T20:14:45,876 INFO [M:0;76919af20f29:34091 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=256.66 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f92770bc51074bb399d73e69f41020b2 2024-11-26T20:14:45,900 DEBUG [M:0;76919af20f29:34091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0ec37a6b7f244bdd8025d87e390ba77c is 69, key is 76919af20f29,43267,1732652055155/rs:state/1732652058863/Put/seqid=0 2024-11-26T20:14:45,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741876_1052 (size=5442) 2024-11-26T20:14:45,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741876_1052 (size=5442) 2024-11-26T20:14:45,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741876_1052 (size=5442) 2024-11-26T20:14:45,914 INFO [M:0;76919af20f29:34091 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0ec37a6b7f244bdd8025d87e390ba77c 2024-11-26T20:14:45,921 INFO [M:0;76919af20f29:34091 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0ec37a6b7f244bdd8025d87e390ba77c 2024-11-26T20:14:45,936 DEBUG [M:0;76919af20f29:34091 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d28a645cdf88496c9494cdd8baaa0194 is 52, key is load_balancer_on/state:d/1732652079638/Put/seqid=0 2024-11-26T20:14:45,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741877_1053 (size=5058) 2024-11-26T20:14:45,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741877_1053 (size=5058) 2024-11-26T20:14:45,944 INFO [M:0;76919af20f29:34091 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d28a645cdf88496c9494cdd8baaa0194 2024-11-26T20:14:45,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741877_1053 (size=5058) 2024-11-26T20:14:45,952 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/77cd35eadd3a45c083b520fb6c25d1f8 as hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/77cd35eadd3a45c083b520fb6c25d1f8 2024-11-26T20:14:45,959 INFO [M:0;76919af20f29:34091 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/77cd35eadd3a45c083b520fb6c25d1f8, entries=14, sequenceid=592, filesize=5.9 K 2024-11-26T20:14:45,960 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f92770bc51074bb399d73e69f41020b2 as hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f92770bc51074bb399d73e69f41020b2 2024-11-26T20:14:45,967 INFO [M:0;76919af20f29:34091 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f92770bc51074bb399d73e69f41020b2, entries=81, sequenceid=592, filesize=12.6 K 2024-11-26T20:14:45,969 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0ec37a6b7f244bdd8025d87e390ba77c as hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0ec37a6b7f244bdd8025d87e390ba77c 2024-11-26T20:14:45,976 INFO [M:0;76919af20f29:34091 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0ec37a6b7f244bdd8025d87e390ba77c 2024-11-26T20:14:45,976 INFO [M:0;76919af20f29:34091 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0ec37a6b7f244bdd8025d87e390ba77c, entries=3, sequenceid=592, filesize=5.3 K 2024-11-26T20:14:45,977 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d28a645cdf88496c9494cdd8baaa0194 as hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d28a645cdf88496c9494cdd8baaa0194 2024-11-26T20:14:45,984 INFO [M:0;76919af20f29:34091 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46543/user/jenkins/test-data/5f531061-fffb-48ec-9bf2-324b5395af0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d28a645cdf88496c9494cdd8baaa0194, entries=1, sequenceid=592, filesize=4.9 K 2024-11-26T20:14:45,985 INFO [M:0;76919af20f29:34091 {}] regionserver.HRegion(3140): Finished flush of dataSize ~257.93 KB/264123, heapSize ~313.11 KB/320624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 222ms, sequenceid=592, compaction requested=false 2024-11-26T20:14:45,997 INFO [M:0;76919af20f29:34091 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T20:14:45,997 DEBUG [M:0;76919af20f29:34091 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732652085763Disabling compacts and flushes for region at 1732652085763Disabling writes for close at 1732652085763Obtaining lock to block concurrent updates at 1732652085763Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732652085763Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=264123, getHeapSize=320624, getOffHeapSize=0, getCellsCount=733 at 1732652085764 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732652085765 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732652085765Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732652085794 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732652085794Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732652085842 (+48 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732652085862 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732652085862Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732652085882 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732652085899 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732652085899Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732652085921 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732652085936 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732652085936Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6abcb9ac: reopening flushed file at 1732652085950 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ff6e219: reopening flushed file at 1732652085959 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@277ae49e: reopening flushed file at 1732652085967 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@257961fc: reopening flushed file at 1732652085976 (+9 ms)Finished flush of dataSize ~257.93 KB/264123, heapSize ~313.11 KB/320624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 222ms, sequenceid=592, compaction requested=false at 1732652085985 (+9 ms)Writing region close event to WAL at 1732652085997 (+12 ms)Closed at 1732652085997 2024-11-26T20:14:46,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34443 is added to blk_1073741865_1041 (size=176307) 2024-11-26T20:14:46,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36117 is added to blk_1073741865_1041 (size=176307) 2024-11-26T20:14:46,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45553 is added to blk_1073741865_1041 (size=176307) 2024-11-26T20:14:46,003 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T20:14:46,003 INFO [M:0;76919af20f29:34091 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T20:14:46,003 INFO [M:0;76919af20f29:34091 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34091 2024-11-26T20:14:46,004 INFO [M:0;76919af20f29:34091 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T20:14:46,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:14:46,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34091-0x10178c82cdf0000, quorum=127.0.0.1:50191, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T20:14:46,113 INFO [M:0;76919af20f29:34091 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T20:14:46,135 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T20:14:46,139 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T20:14:46,139 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T20:14:46,140 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T20:14:46,140 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir/,STOPPED} 2024-11-26T20:14:46,146 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T20:14:46,146 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T20:14:46,146 WARN [BP-1003607189-172.17.0.3-1732652041092 heartbeating to localhost/127.0.0.1:46543 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T20:14:46,146 WARN [BP-1003607189-172.17.0.3-1732652041092 heartbeating to localhost/127.0.0.1:46543 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1003607189-172.17.0.3-1732652041092 (Datanode Uuid 36b267de-6be8-493c-ae7d-2084bce59074) service to localhost/127.0.0.1:46543 2024-11-26T20:14:46,148 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data5/current/BP-1003607189-172.17.0.3-1732652041092 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T20:14:46,149 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data6/current/BP-1003607189-172.17.0.3-1732652041092 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T20:14:46,150 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T20:14:46,166 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T20:14:46,168 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T20:14:46,168 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T20:14:46,168 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T20:14:46,168 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir/,STOPPED} 2024-11-26T20:14:46,180 WARN [BP-1003607189-172.17.0.3-1732652041092 heartbeating to localhost/127.0.0.1:46543 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T20:14:46,180 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T20:14:46,180 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T20:14:46,180 WARN [BP-1003607189-172.17.0.3-1732652041092 heartbeating to localhost/127.0.0.1:46543 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1003607189-172.17.0.3-1732652041092 (Datanode Uuid 3d7e7ac9-d193-4317-a686-8f41042da5ab) service to localhost/127.0.0.1:46543 2024-11-26T20:14:46,181 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data3/current/BP-1003607189-172.17.0.3-1732652041092 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T20:14:46,182 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T20:14:46,184 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data4/current/BP-1003607189-172.17.0.3-1732652041092 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T20:14:46,209 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T20:14:46,211 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T20:14:46,211 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T20:14:46,223 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T20:14:46,223 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir/,STOPPED} 2024-11-26T20:14:46,228 WARN [BP-1003607189-172.17.0.3-1732652041092 heartbeating to localhost/127.0.0.1:46543 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T20:14:46,228 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T20:14:46,229 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T20:14:46,229 WARN [BP-1003607189-172.17.0.3-1732652041092 heartbeating to localhost/127.0.0.1:46543 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1003607189-172.17.0.3-1732652041092 (Datanode Uuid 36f9bb8a-7dcf-4d56-8314-0ba5daeef0f4) service to localhost/127.0.0.1:46543 2024-11-26T20:14:46,230 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data2/current/BP-1003607189-172.17.0.3-1732652041092 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T20:14:46,232 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/cluster_59ca703c-c6a3-7b31-4925-707777b591e1/data/data1/current/BP-1003607189-172.17.0.3-1732652041092 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T20:14:46,236 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T20:14:46,251 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T20:14:46,253 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T20:14:46,253 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T20:14:46,254 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T20:14:46,254 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/107b2ea1-f292-91c7-906e-950d0269f18d/hadoop.log.dir/,STOPPED} 2024-11-26T20:14:46,278 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T20:14:46,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down